@microsoft/m365-copilot-eval 1.0.1-preview.1

package/src/clients/cli/main.py

@@ -0,0 +1,766 @@
+import json
+import os
+import argparse
+import sys
+import csv
+import webbrowser
+import urllib.request
+import urllib.error
+import urllib.parse
+import questionary
+from enum import Enum
+from typing import List, Dict, Tuple, Optional, Any
+from azure.ai.evaluation import (
+    AzureOpenAIModelConfiguration,
+    RelevanceEvaluator,
+    CoherenceEvaluator,
+    GroundednessEvaluator,
+    ToolCallAccuracyEvaluator
+)
+from dotenv import load_dotenv
+from auth.auth_handler import AuthHandler
+from custom_evaluators.CitationsEvaluator import CitationsEvaluator, CitationFormat
+#from custom_evaluators.ConcisenessNonLLMEvaluator import ConcisenessNonLLMEvaluator
+#from custom_evaluators.PII.PII import PIIEvaluator
+from generate_report import generate_html_report, calculate_aggregate_statistics
+from response_extractor import extract_enhanced_responses, get_response_text_for_evaluation
+from datetime import datetime, timezone
+
+# Allowed endpoints for URL validation
+ALLOWED_ENDPOINTS = [
+    'substrate.office.com'
+]
+
+class CallPath(Enum):
+    """ Enum to indicate which call path to use. """
+    ACCESS_TOKEN = "access_token"
+    COPILOT_AUTH = "copilot_auth"
+
+def write_results_to_html(results: List[Dict], output_file: str):
+    """Write results to HTML file using generate_html_report from generate_report.py."""
+    try:
+        html = generate_html_report(results) 
+        with open(output_file, 'w', encoding='utf-8') as f:
+            f.write(html)
+        print(f"HTML report saved to {output_file}")
+    except Exception as e:
+        print(f"Error writing to HTML file: {e}")
+        sys.exit(1)
+
+def get_default_prompts_and_responses():
+    """Get a list of prompts and responses."""
+    prompts = [
+        "What is Microsoft Graph?"
+    ]
+    expected_responses = [
+        "Microsoft Graph is a gateway to data and intelligence in Microsoft 365."
+    ]
+    return prompts, expected_responses
+
+def load_prompts_from_file(file_path: str) -> Tuple[List[str], List[str]]:
+    """Load prompts and expected responses from a JSON file."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        
+        if isinstance(data, list):
+            # Format: [{"prompt": "...", "expected_response": "..."}, ...]
+            prompts = [item.get("prompt", "") for item in data]
+            expected_responses = [item.get("expected_response", "") for item in data]
+        elif isinstance(data, dict):
+            # Format: {"prompts": [...], "expected_responses": [...]}
+            prompts = data.get("prompts", [])
+            expected_responses = data.get("expected_responses", [])
+        else:
+            raise ValueError("Invalid file format")
+        
+        return prompts, expected_responses
+    except Exception as e:
+        print(f"Error loading prompts from file: {e}")
+        sys.exit(1)
+
+def get_interactive_prompts() -> Tuple[List[str], List[str]]:
+    """Get prompts and expected responses interactively."""
+    prompts = []
+    expected_responses = []
+    
+    print("Interactive mode: Enter your prompts and expected responses.")
+    print("Press Enter with empty prompt to finish.")
+    
+    while True:
+        prompt = input(f"\nPrompt {len(prompts) + 1}: ").strip()
+        if not prompt:
+            break
+        
+        expected = input(f"Expected response {len(expected_responses) + 1}: ").strip()
+        
+        prompts.append(prompt)
+        expected_responses.append(expected)
+    
+    if not prompts:
+        print("No prompts entered. Exiting.")
+        sys.exit(1)
+    
+    return prompts, expected_responses
+
+def run_evaluations(args, responses: dict, expected_responses: list) -> list:
+    """Run evaluations against the responses."""
+    model_config = AzureOpenAIModelConfiguration(
+        azure_endpoint=os.environ.get("AZURE_AI_OPENAI_ENDPOINT"),
+        api_key=os.environ.get("AZURE_AI_API_KEY"),
+        api_version=os.environ.get("AZURE_AI_API_VERSION"),
+        azure_deployment=os.environ.get("AZURE_AI_MODEL_NAME"),
+    )
+    
+    # Initialize evaluators
+    relevance_evaluator = RelevanceEvaluator(model_config=model_config) # Evaluate relevance for a given response. Range is 1 - 5.
+    coherence_evaluator = CoherenceEvaluator(model_config=model_config) # Measures the coherence (human-like quality) of the response. Range is 1 - 5.
+    groundedness_evaluator = GroundednessEvaluator(model_config=model_config) # Evaluates the response for for factuality and groundedness against provided context. Range is 1 - 5.
+    #concisenessnonllm_evaluator = ConcisenessNonLLMEvaluator() # Evaluates the response for conciseness. Range is 1 - 5.
+    #pii_evaluator = PIIEvaluator(model_config=model_config) # Evaluates the response for presence of PII. Range
+    # Parse citation format from args
+    citation_format = CitationFormat.OAI_UNICODE if args.citation_format == 'oai_unicode' else CitationFormat.LEGACY_BRACKET
+    citations_evaluator = CitationsEvaluator(citation_format=citation_format) # Evaluates citations present in the response using regex pattern matching
+
+    tool_call_accuracy_evaluator = ToolCallAccuracyEvaluator(model_config) # Evaluate tool call accuracy if tool definitions are present in response
+    
+
+    PASS_THRESHOLD = 3  # All evaluators must meet or exceed this value (out of 5) to pass
+
+    def decorate_metric(metric_id: str, data):
+        """Augment raw evaluator output with standardized threshold + pass/fail result."""
+        payload = {}
+        # Preserve original structure if dict
+        if isinstance(data, dict):
+            payload.update(data)
+        else:
+            payload['raw'] = data
+
+        # Try to extract a numeric score
+        score_val = None
+        if isinstance(data, dict):
+            for k in (metric_id, f"{metric_id}_score", 'score', 'value'):
+                if k in data:
+                    score_val = data[k]
+                    break
+        if isinstance(score_val, (int, float)):
+            payload['threshold'] = PASS_THRESHOLD
+            payload['result'] = 'pass' if score_val >= PASS_THRESHOLD else 'fail'
+        else:
+            # If we cannot determine score, mark unknown (no pass/fail)
+            payload['threshold'] = PASS_THRESHOLD
+            payload.setdefault('result', 'unknown')
+        return json.dumps(payload, indent=4)
+
+    evaluation_results = []
+    for prompt, expected_response in zip(responses.keys(), expected_responses):
+        # Extract text response for evaluation (backward compatibility)
+        enhanced_response = responses[prompt]
+        actual_response_text = get_response_text_for_evaluation(enhanced_response)
+        
+        # Run evaluations using text response
+        relevance_score = relevance_evaluator(
+            query=prompt,
+            response=actual_response_text
+        )
+        coherence_score = coherence_evaluator(
+            query=prompt,
+            response=actual_response_text
+        )
+
+        groundedness_score = groundedness_evaluator(
+            response=actual_response_text,
+            context=expected_response
+        )
+
+        #PII_score = pii_evaluator(response=actual_response_text)
+        #concisenessNonLLM_score = concisenessnonllm_evaluator(response=actual_response_text)
+
+        citations_score = citations_evaluator(
+            response=actual_response_text
+        )
+
+        tool_call_accuracy = None
+        if args.agent_id and enhanced_response.get("tool_definitions"):
+            tool_call_accuracy = tool_call_accuracy_evaluator(
+                query=prompt,
+                response=enhanced_response.get("response", actual_response_text),
+                tool_definitions=enhanced_response["tool_definitions"]
+            )
+
+        evaluation_result = {
+            "prompt": prompt,
+            "response": actual_response_text,  # Keep simple text for backward compatibility
+            "expected_response": expected_response,
+            "results": {
+                "relevance_score": decorate_metric("relevance", relevance_score),
+                "coherence_score": decorate_metric("coherence", coherence_score),
+                "groundedness_score": decorate_metric("groundedness", groundedness_score),
+                #"concisenessnonllm_score": decorate_metric("concisenessnonllm", concisenessNonLLM_score),
+                #"pii_score": decorate_metric("pii", PII_score),
+                "citations_score": json.dumps(citations_score, indent=4),
+                "tool_call_accuracy_score": json.dumps(tool_call_accuracy, indent=4) if tool_call_accuracy else None
+            }
+        }
+
+        if args.verbose:
+            print(f".................................. Evaluation for prompt: {evaluation_result['prompt']} ..................................")
+            print(f"Scores: {evaluation_result['results']}")
+            print("...........................................................................................................................")
+
+        evaluation_results.append(evaluation_result)
+    
+    return evaluation_results
+
+def write_results_to_console(results):
+    """Write the response to console."""
+    # ANSI color codes
+    BOLD = '\033[1m'
+    BLUE = '\033[94m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    CYAN = '\033[96m'
+    MAGENTA = '\033[95m'
+    ORANGE = '\033[38;5;208m'
+    RED = '\033[91m'
+    RESET = '\033[0m'
+    
+    # Show aggregate statistics if multiple results
+    if len(results) > 1:
+        aggregates = calculate_aggregate_statistics(results)
+        if aggregates:
+            print(f"{BOLD}{BLUE}📊 Aggregate Statistics ({len(results)} prompts):{RESET}")
+            print(f"{BLUE}{'=' * 60}{RESET}")
+            
+            for metric_name, stats in aggregates.items():
+                pass_color = GREEN if stats['pass_rate'] >= 80 else YELLOW if stats['pass_rate'] >= 60 else RED
+                print(f"{BOLD}{CYAN}{metric_name}:{RESET}")
+                print(f"  Pass Rate: {pass_color}{stats['pass_rate']:.1f}%{RESET} ({stats['pass_count']}/{stats['total_evaluated']} passed)")
+                print(f"  Avg Score: {MAGENTA}{stats['avg_score']:.2f}{RESET}")
+                if stats.get('threshold') is not None:
+                    print(f"  Threshold: {YELLOW}{stats['threshold']}{RESET}")
+                print()
+            
+            print(f"{BLUE}{'=' * 60}{RESET}")
+            print()
+    
+    print(f"{BOLD}{BLUE}📝 Individual Results:{RESET}")
+    print(f"{BLUE}{'=' * 50}{RESET}")
+    for i, result in enumerate(results, 1):
+        print(f"{BOLD}{GREEN}Prompt {i}:{RESET} {result['prompt']}")
+        print(f"{BOLD}{CYAN}Response:{RESET} {result['response']}")
+        print(f"{BOLD}{YELLOW}Expected Response:{RESET} {result['expected_response']}")
+
+        # Print metric scores generically from nested results (fallback to flat keys for back-compat)
+        metrics = result.get('results') or {k: v for k, v in result.items() if isinstance(k, str) and k.endswith('_score')}
+        if metrics:
+            for k, v in metrics.items():
+                name = k.replace('_', ' ')
+                if 'relevance' in k:
+                    color = MAGENTA
+                elif 'coherence' in k:
+                    color = ORANGE
+                elif 'fluency' in k:
+                    color = GREEN
+                else:
+                    color = BLUE
+                print(f"{BOLD}{color}{name}:{RESET} {v}")
+        print(f"{BLUE}{'-' * 30}{RESET}")
+
+def write_results_to_json(results: List[Dict], output_file: str):
+    """Write results to JSON file."""
+    try:
+        output_data = {
+            "individual_results": results
+        }
+        
+        # Add aggregate statistics if multiple results
+        if len(results) > 1:
+            aggregates = calculate_aggregate_statistics(results)
+            output_data["aggregate_statistics"] = aggregates
+            
+        with open(output_file, 'w', encoding='utf-8') as f:
+            json.dump(output_data, f, indent=2, ensure_ascii=False)
+        print(f"Results saved to {output_file}")
+    except Exception as e:
+        print(f"Error writing to JSON file: {e}")
+        sys.exit(1)
+
+def write_results_to_csv(results: List[Dict], output_file: str):
+    """Write results to CSV file."""
+    try:
+        with open(output_file, 'w', newline='', encoding='utf-8') as f:
+            if results:
+                # Write aggregate statistics first if multiple results
+                if len(results) > 1:
+                    aggregates = calculate_aggregate_statistics(results)
+                    if aggregates:
+                        f.write("# AGGREGATE STATISTICS\n")
+                        f.write("Metric,Pass Rate (%),Passed,Failed,Avg Score,Threshold\n")
+                        for metric_name, stats in aggregates.items():
+                            threshold_str = str(stats.get('threshold', 'N/A'))
+                            f.write(f"{metric_name},{stats['pass_rate']:.1f},{stats['pass_count']},{stats['fail_count']},{stats['avg_score']:.2f},{threshold_str}\n")
+                        f.write("\n# INDIVIDUAL RESULTS\n")
+                
+                # Write individual results
+                writer = csv.DictWriter(f, fieldnames=results[0].keys())
+                writer.writeheader()
+                writer.writerows(results)
+        print(f"Results saved to {output_file}")
+    except Exception as e:
+        print(f"Error writing to CSV file: {e}")
+        sys.exit(1)
+
+def parse_arguments():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="M365 Copilot Agent Evaluation CLI",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run with default prompts
+  python main.py
+
+  # Run with custom prompts
+  python main.py --prompts "What is Microsoft Graph?" --expected "Microsoft Graph is a gateway..."
+
+  # Run with prompts from file
+  python main.py --prompts-file prompts.json
+
+  # Interactive mode
+  python main.py --interactive
+
+  # Save results to JSON
+  python main.py --output results.json
+
+  # Save results to CSV
+  python main.py --output results.csv
+
+  # Save results to HTML and open in browser
+  python main.py --output report.html
+
+  # Verbose output
+  python main.py --verbose
+
+  # Sign out and clear cached authentication tokens
+  python main.py --signout
+        """
+    )
+    
+    # Input options (mutually exclusive)
+    input_group = parser.add_mutually_exclusive_group()
+    input_group.add_argument(
+        '--prompts', 
+        nargs='+', 
+        help='List of prompts to evaluate'
+    )
+    input_group.add_argument(
+        '--prompts-file', 
+        type=str, 
+        help='JSON file containing prompts and expected responses'
+    )
+    input_group.add_argument(
+        '--interactive', 
+        action='store_true', 
+        help='Interactive mode to enter prompts'
+    )
+    
+    # Expected responses (only used with --prompts)
+    parser.add_argument(
+        '--expected', 
+        nargs='+', 
+        help='List of expected responses (must match number of prompts)'
+    )
+    
+    # Agent ID
+    parser.add_argument(
+        '--agent-id', 
+        type=str, 
+        default=os.environ.get("AGENT_ID"),
+        help='Azure AI Agent ID (default from environment variable)'
+    )
+
+    # Output options
+    parser.add_argument(
+        '--output', 
+        type=str, 
+        help='Output file path. Format is determined by file extension: .json, .csv, .html. If not provided, results are printed to console.'
+    )
+    
+    # Behavior options
+    parser.add_argument(
+        '--verbose', 
+        action='store_true', 
+        help='Enable verbose output'
+    )
+    parser.add_argument(
+        '--quiet', 
+        action='store_true', 
+        help='Suppress non-essential output'
+    )
+    parser.add_argument(
+        '--citation-format',
+        choices=['oai_unicode', 'legacy_bracket'],
+        default='oai_unicode',
+        help='Citation format to detect. "oai_unicode" for new OAI format (default), "legacy_bracket" for old [^i^] format'
+    )
+    parser.add_argument(
+        '--signout',
+        action='store_true',
+        help='Sign out and clear cached authentication tokens'
+    )
+    
+    return parser.parse_args()
+
+def validate_environment() -> CallPath:
+    """Validate required environment variables."""
+    required_env_vars = [
+        "AZURE_AI_OPENAI_ENDPOINT",
+        "AZURE_AI_API_KEY",
+        "AZURE_AI_API_VERSION",
+        "AZURE_AI_MODEL_NAME",
+        # Chat API specific
+        "COPILOT_API_ENDPOINT",
+        "X_SCENARIO_HEADER"
+    ]
+
+    if os.environ.get("COPILOT_API_ACCESS_TOKEN"):
+        call_path = CallPath.ACCESS_TOKEN
+        required_env_vars.append("COPILOT_API_ACCESS_TOKEN")
+    else:
+        call_path = CallPath.COPILOT_AUTH
+        required_env_vars.extend([
+            "M365_EVAL_CLIENT_ID",
+            "TENANT_ID"
+        ])
+    
+    missing_vars = [var for var in required_env_vars if not os.environ.get(var)]
+    if missing_vars:
+        print(f"Error: Missing required environment variables: {', '.join(missing_vars)}")
+        print("Please ensure your .env file contains all required Azure configuration.")
+        sys.exit(1)
+    return call_path
+
+def validate_endpoint_url(url: str, allowed_domains: List[str]) -> bool:
+    """Validate URL against security requirements."""
+    try:
+        parsed = urllib.parse.urlparse(url)
+        
+        # Check for dangerous schemes
+        if parsed.scheme in ['javascript', 'data']:
+            raise ValueError(f"Dangerous URL scheme detected: {parsed.scheme}")
+        
+        # Check for HTTPS requirement
+        if parsed.scheme != 'https':
+            raise ValueError(f"Only HTTPS URLs are allowed, got: {parsed.scheme}")
+        
+        # Check if domain is in allowed list
+        if parsed.netloc not in allowed_domains:
+            raise ValueError(f"Domain not in allowed list: {parsed.netloc}")
+        
+        # Reject fragment URLs
+        if parsed.fragment:
+            raise ValueError("Fragment URLs are not allowed")
+        
+        return True
+        
+    except ValueError:
+        # Re-raise ValueError exceptions
+        raise
+    except Exception as e:
+        # Convert other parsing errors to ValueError
+        raise ValueError(f"Invalid URL format: {url}") from e
+
+def get_prompt_datasets(args) -> Tuple[List[str], List[str]]:
+    """Get prompts and expected responses based on command line arguments."""
+    if args.prompts:
+        if args.expected and len(args.prompts) != len(args.expected):
+            print("Error: Number of prompts must match number of expected responses")
+            sys.exit(1)
+        prompts = args.prompts
+        expected_responses = args.expected or [""] * len(prompts)
+    elif args.prompts_file:
+        prompts, expected_responses = load_prompts_from_file(args.prompts_file)
+    elif args.interactive:
+        prompts, expected_responses = get_interactive_prompts()
+    else:
+        # Use default prompts
+        prompts, expected_responses = get_default_prompts_and_responses()
+    
+    return prompts, expected_responses
+
+def fetch_available_agents(copilot_api_endpoint: str, access_token: str, user_oid: str) -> List[Dict[str, Any]]:
+    """
+    Fetch available agents for the user from the Copilot API.
+    
+    Args:
+        access_token: Bearer token for API authentication
+        user_oid: User object ID for agent filtering
+        
+    Returns:
+        List of agent dictionaries.
+    """
+    request_headers = {
+        "Content-Type": "application/json",
+        "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
+        "Authorization": f"Bearer {access_token}"
+    }
+    
+    try:
+        # Build the query parameter with participant info
+        request_data = json.dumps({"participant": {"id": user_oid}})
+        query_param = urllib.parse.quote(request_data)
+        
+        # Try to fetch agents from /GetGptList endpoint
+        req = urllib.request.Request(
+            f"{copilot_api_endpoint}/GetGptList?request={query_param}",
+            headers=request_headers,
+            method="GET"
+        )
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            data = json.loads(resp.read().decode("utf-8"))
+            agents = data.get("gptList", [])
+            return agents
+    except urllib.error.HTTPError as e:
+        # If endpoint doesn't exist or returns error, return empty list
+        print(f"Warning: Unable to fetch agents list (HTTP {e.code}).")
+        return []
+    except Exception as e:
+        print(f"Warning: Error fetching agents: {e}")
+        return []
+
+def select_agent_interactively(agents: List[Dict[str, Any]]) -> Optional[str]:
+    """
+    Display an interactive agent selector using questionary.
+    
+    Args:
+        agents: List of agent dictionaries.
+        
+    Returns:
+        Selected agent ID or None if cancelled/skipped
+    """
+    if not agents:
+        return None
+    
+    # Create choices for questionary
+    choices = []
+    sorted_agents = sorted(agents, key=lambda x: (not x.get('isOwner', False), x.get('name', '')))
+    for agent in sorted_agents:
+        agent_id = agent.get("gptId", "Unknown")
+        agent_name = agent.get("name", "Unknown")
+        agent_description = agent.get("description", "Unknown")
+        agent_is_owner = agent.get('isOwner')
+        
+        # Format the display text
+        display_text = f"{agent_name} ({agent_id}, IsOwner: {agent_is_owner}) - {agent_description}"
+        
+        choices.append(questionary.Choice(title=display_text, value=agent_id))
+    
+    # Display the selection prompt
+    selected_agent = questionary.select(
+        "Select an agent to evaluate:",
+        choices=choices,
+        use_shortcuts=True,
+        use_arrow_keys=True
+    ).ask()
+    
+    return selected_agent
+
+def build_chat_payload(prompt: str, user_oid: str, agent_id: str|None) -> bytes:
+    message = {
+        "message": {
+            "text": prompt,
+            "author": "user",
+            "messageType": "chat",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "from": {
+                    "id": user_oid,
+            }
+        },
+        "verbosity": "verbose", # To enable detailed telemetry in response (to extract tool usage, etc.)
+    }
+    
+    if agent_id:
+            message["gpts"] = [
+                {
+                        "id": agent_id.strip(),
+                        "source": "MOS3"
+                }
+            ]
+            message["optionsSets"] = [
+                "disable_action_confirmation"   # Disable 3P action confirmation prompts for agents while scraping
+            ]
+  
+    return json.dumps(message).encode("utf-8")
+
+def send_prompt_to_agent_in_sydney(prompts: List[str], copilot_api_endpoint: str, access_token: str, user_oid: str, args) -> Dict[str, Dict[str, any]]:
+    """ Send prompts to the chat API and return enhanced responses. """
+
+    request_headers = {
+        "Content-Type": "application/json",
+        "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
+        "Authorization": f"Bearer {access_token}"
+        }
+    raw_responses: Dict[str, str] = {}
+    for i, prompt in enumerate(prompts, 1):
+        if not args.quiet:
+            print(f"Processing prompt {i}/{len(prompts)}...")
+
+        # Build the payload
+        payload = build_chat_payload(prompt, user_oid, args.agent_id)
+        if args.verbose:
+            print(f"[Sydney] Sending payload: {payload.decode('utf-8')}")
+
+        # Send the request to /chat
+        req = urllib.request.Request(f"{copilot_api_endpoint}/chat", data=payload, headers=request_headers, method="POST")
+        try:
+            with urllib.request.urlopen(req, timeout=120) as resp:
+                raw = resp.read().decode("utf-8", errors="replace")
+        except urllib.error.HTTPError as e:
+            error_body = None
+            try:
+                error_body = e.read().decode("utf-8", errors="replace")
+            except Exception:
+                pass
+            msg = f"Chat API request failed (HTTP {e.code} {e.reason})."
+            if error_body:
+                msg += f" Body: {error_body[:500]}"
+            raise RuntimeError(msg) from e
+        except urllib.error.URLError as e:
+            raise RuntimeError(f"Chat API connection error: {getattr(e, 'reason', str(e))}") from e
+        
+        if args.verbose:
+            print(f"[Sydney] Raw response: {raw}")
+        
+        # Store raw response for enhancement
+        raw_responses[prompt] = raw.strip()
+    
+    # Extract enhanced responses using the new extractor
+    enhanced_responses = extract_enhanced_responses(raw_responses)
+    return enhanced_responses
+
+def output_results(results: List[Dict], args):
+    """Output results based on specified format."""
+    if args.output:
+        output_lower = args.output.lower()
+        if output_lower.endswith('.json'):
+            write_results_to_json(results, args.output)
+        elif output_lower.endswith('.csv'):
+            write_results_to_csv(results, args.output)
+        elif output_lower.endswith('.html'):
+            write_results_to_html(results, args.output)
+            abs_path = os.path.abspath(args.output)
+            webbrowser.open(f'file://{abs_path}')
+        else:
+            write_results_to_json(results, args.output)
+    else:
+        write_results_to_console(results)
+
+def main():
+    """Main function to orchestrate the evaluation process."""
+    load_dotenv()
+    args = parse_arguments()
+
+    # Validate environment variables required for evaluation
+    call_path = validate_environment()
+    copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
+    validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
+    
+    user_oid = ""
+
+    if call_path == CallPath.ACCESS_TOKEN:
+        access_token = os.environ["COPILOT_API_ACCESS_TOKEN"]
+    else:
+      scopes_str = os.environ.get(
+          "COPILOT_SCOPES", "https://substrate.office.com/sydney/.default"
+          )
+          
+      auth_handler = AuthHandler(
+          client_id=os.environ["M365_EVAL_CLIENT_ID"],
+          tenant_id=os.environ["TENANT_ID"],
+          scopes_str=scopes_str
+          )
+
+      # Signout user
+      if args.signout:
+          try:
+              auth_handler.clear_cache()
+          except Exception as e:
+              print(f"Error during signout: {e}")
+              sys.exit(1)
+          sys.exit(0)
+
+      # Authenticate before loading prompts
+      try:
+        auth_result = auth_handler.acquire_token_interactive() or {}
+        access_token = auth_result.get("access_token") or ""
+        if not access_token:
+            raise RuntimeError("Failed to acquire access token from authentication result")
+
+        id_token_claims = auth_result.get("id_token_claims")
+        if not isinstance(id_token_claims, dict):
+            print("id_token_claims is missing or invalid in authentication result")
+        else:
+            user_oid = id_token_claims.get("oid") or ""
+        
+      except Exception as e:
+        print(f"\033[91mError during authentication: {e}\033[0m")
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+
+    if not user_oid and access_token:
+        # Fallback: extract from access token.
+        user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
+
+    # 1. Load evaluation datasets (prompts and expected_responses)
+    prompts, expected_responses = get_prompt_datasets(args)
+
+    if not args.quiet:
+        print(f"Running evaluation on {len(prompts)} prompt(s)...")
+
+    try:
+        # 3. Agent selection - if no agent ID provided, prompt user to select
+        if not args.agent_id:
+            if not args.quiet:
+                print("No agent ID provided. Fetching available agents...")
+            
+            available_agents = fetch_available_agents(copilot_api_endpoint, access_token, user_oid)
+            if not available_agents:
+              print("No agents are available for interactive selection. Please re-run with --agent-id or set the AGENT_ID environment variable.")
+              sys.exit(1)
+            
+            if available_agents:
+                selected_agent_id = select_agent_interactively(available_agents)
+                if selected_agent_id:
+                    args.agent_id = selected_agent_id
+                    if not args.quiet:
+                        print(f"Selected agent: {args.agent_id}")
+                else:
+                    print("No agent selected. Please re-run with --agent-id or set the AGENT_ID environment variable.")
+                    sys.exit(1)
+
+        # 4. Send prompts to chat API
+        responses = send_prompt_to_agent_in_sydney(prompts, copilot_api_endpoint, access_token, user_oid, args)
+    except Exception as e:
+        print(f"\033[91mError sending prompts to chat API: {e}\033[0m")
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+    
+    # 5. Run evaluations
+    if not args.quiet:
+        print("Running evaluations...")
+    results = run_evaluations(args, responses, expected_responses)
+    
+    # 6. Output results
+    output_results(results, args)
+    
+    if not args.quiet:
+        print(f"\nEvaluation completed successfully! Processed {len(prompts)} prompt(s).")
+
+# Call the main function when script is run directly
+if __name__ == "__main__":
+    main()