@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.5.0-preview.1

package/src/clients/cli/main.py

@@ -1,1097 +1,57 @@
-import json
+"""M365 Copilot Agent Evaluation CLI — thin orchestrator.
+
+Delegates to focused modules:
+  cli_args          – argument parsing & version-check bypass
+  env_validator     – environment validation & URL security
+  prompt_loader     – dataset loading & agent selection
+  evaluation_runner – pipeline, evaluator dispatch, retry
+  result_writer     – console / JSON / CSV / HTML output
+"""
+
 import os
-import argparse
 import sys
-import csv
-import functools
-import logging
-import webbrowser
-import urllib.request
-import urllib.error
-import urllib.parse
-import questionary
-from enum import Enum
-from typing import List, Dict, Tuple, Optional, Any
-from azure.ai.evaluation import (
-    AzureOpenAIModelConfiguration,
-    RelevanceEvaluator,
-    CoherenceEvaluator,
-    GroundednessEvaluator,
-    ToolCallAccuracyEvaluator
-)
+import traceback
+
+from azure.ai.evaluation import AzureOpenAIModelConfiguration
 from dotenv import load_dotenv
+
+from api_clients.A2A import A2AClient
 from auth.auth_handler import AuthHandler
-from custom_evaluators.CitationsEvaluator import CitationsEvaluator, CitationFormat
-from custom_evaluators.ExactMatchEvaluator import ExactMatchEvaluator
-from custom_evaluators.PartialMatchEvaluator import PartialMatchEvaluator
-from generate_report import generate_html_report, calculate_aggregate_statistics
-from response_extractor import extract_enhanced_responses, get_response_text_for_evaluation
-from schema_handler import DocumentUpgrader, SchemaVersionManager
-from common import (
-    RELEVANCE,
-    COHERENCE,
-    GROUNDEDNESS,
-    TOOL_CALL_ACCURACY,
-    CITATIONS,
-    EXACT_MATCH,
-    PARTIAL_MATCH,
-    REQUIRES_AZURE_OPENAI,
-    REQUIRES_TOOL_DEFINITIONS,
-    METRIC_IDS,
-    pascal_case_to_title,
-)
-from evaluator_resolver import (
-    EVALUATOR_REGISTRY,
-    validate_evaluator_names,
-    check_prerequisites,
-    resolve_default_evaluators,
-    resolve_evaluators_for_prompt,
-    get_evaluator_threshold,
-)
+from evaluator_resolver import resolve_default_evaluators
 from version_check import check_min_version, get_cli_version
-from datetime import datetime, timezone
-from pathlib import Path
-import tzlocal
 
-from cli_logging.console_diagnostics import render_diagnostic, serialize_diagnostic_record
-from cli_logging.logging_utils import LOG_LEVEL_MAP, LogLevel, Operation, format_structured_log_entry, resolve_log_level
-
-# Allowed endpoints for URL validation
-ALLOWED_ENDPOINTS = [
-    'substrate.office.com'
-]
-
-class CallPath(Enum):
-    """ Enum to indicate which call path to use. """
-    ACCESS_TOKEN = "access_token"
-    COPILOT_AUTH = "copilot_auth"
-
-
-# Flags that should bypass remote min-version enforcement.
-# --help is not needed here because argparse exits before runtime checks.
-VERSION_CHECK_BYPASS_FLAGS = (
-    "signout",
+from cli_logging.cli_logger import (
+    CLI_LOGGER,
+    DIAGNOSTIC_RECORDS,
+    configure_cli_logging,
+    emit_structured_log,
 )
+from cli_logging.logging_utils import Operation, resolve_log_level
 
-CLI_LOGGER_NAME = "m365.eval.cli"
-CLI_LOGGER = logging.getLogger(CLI_LOGGER_NAME)
-DIAGNOSTIC_RECORDS: List[Dict[str, Any]] = []
-
-
-def _ensure_logger_handler() -> None:
-    if CLI_LOGGER.handlers:
-        return
-    handler = logging.StreamHandler(sys.stdout)
-    handler.setFormatter(logging.Formatter("%(message)s"))
-    CLI_LOGGER.addHandler(handler)
-    CLI_LOGGER.propagate = False
-
-
-def configure_cli_logging(effective_log_level: str) -> None:
-    _ensure_logger_handler()
-    CLI_LOGGER.setLevel(LOG_LEVEL_MAP[effective_log_level])
-
-
-def emit_structured_log(level: str, message: str, operation: str = Operation.EVALUATE) -> None:
-    _ensure_logger_handler()
-    context = {
-        "request-id": None,
-        "conversation-id": None,
-        "message-id": None,
-        "operation": operation,
-    }
-    entry = format_structured_log_entry(
-        level=level,
-        message=message,
-        logger_name=CLI_LOGGER_NAME,
-        run_context=context,
-    )
-    DIAGNOSTIC_RECORDS.append(entry)
-
-    try:
-        CLI_LOGGER.log(LOG_LEVEL_MAP.get(level, logging.INFO), render_diagnostic(entry))
-    except Exception:
-        pass
-
-
-def should_bypass_min_version_check(args: argparse.Namespace) -> bool:
-    """Return True if the current invocation should skip min-version checks."""
-    return any(getattr(args, flag, False) for flag in VERSION_CHECK_BYPASS_FLAGS)
-
-def write_results_to_html(results: List[Dict], output_file: str,
-                          agent_name: Optional[str] = None, agent_id: Optional[str] = None,
-                          cli_version: Optional[str] = None):
-    """Write results to HTML file using generate_html_report from generate_report.py."""
-    try:
-        html = generate_html_report(results, agent_name=agent_name, agent_id=agent_id,
-                                    cli_version=cli_version)
-        with open(output_file, 'w', encoding='utf-8') as f:
-            f.write(html)
-        emit_structured_log("info", f"HTML report saved to {output_file}", operation=Operation.WRITE_OUTPUT)
-    except Exception as e:
-        emit_structured_log("error", f"Error writing to HTML file: {e}", operation=Operation.WRITE_OUTPUT)
-        sys.exit(1)
-
-def get_default_prompts_and_responses():
-    """Get a list of prompts and responses."""
-    prompts = [
-        "What is Microsoft Graph?"
-    ]
-    expected_responses = [
-        "Microsoft Graph is a gateway to data and intelligence in Microsoft 365."
-    ]
-    return prompts, expected_responses
-
-def load_prompts_from_file(file_path: str) -> Tuple[List[Dict], Optional[Dict]]:
-    """Load prompts and expected responses from a JSON file.
-
-    Supports three formats:
-    1. Eval document: {"schemaVersion": "1.0.0", "items": [{"prompt": "..."}]}
-    2. Array format: [{"prompt": "...", "expected_response": "..."}]
-    3. Dict format: {"prompts": [...], "expected_responses": [...]}
-
-    For eval documents (format 1) and array format (format 2), schema validation
-    and auto-upgrade are applied via DocumentUpgrader.
-
-    Returns:
-        Tuple of (eval_items, default_evaluators). Items are dicts with prompt,
-        expected_response, and optional evaluators/evaluators_mode fields.
-    """
-    try:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            data = json.load(f)
-
-        # Detect if this is an eval document (has "items" key) or could be upgraded
-        is_eval_document = (
-            isinstance(data, dict) and "items" in data
-        ) or isinstance(data, list)
-
-        # Run schema validation and auto-upgrade for eval documents
-        if is_eval_document:
-            try:
-                upgrader = DocumentUpgrader()
-            except Exception as e:
-                # Schema infrastructure not available (missing files, etc.) — skip
-                emit_structured_log("warning", f"Unable to initialize document upgrader: {e}", operation=Operation.LOAD_PROMPTS)
-                upgrader = None
-
-            if upgrader is not None:
-                result = upgrader.upgrade(Path(file_path))
-
-                if result.error:
-                    emit_structured_log("error", f"Schema validation error: {result.error}", operation=Operation.LOAD_PROMPTS)
-                    sys.exit(1)
-
-                if result.upgraded and result.message:
-                    emit_structured_log("info", result.message, operation=Operation.LOAD_PROMPTS)
-
-                # Use the parsed document from the upgrade result
-                if result.document is not None:
-                    data = result.document
-
-        if isinstance(data, list):
-            # Format: [{"prompt": "...", "expected_response": "..."}, ...]
-            return data, None
-        elif isinstance(data, dict):
-            if "items" in data:
-                # Eval document format: {"schemaVersion": "...", "items": [...]}
-                return data["items"], data.get("default_evaluators")
-            else:
-                # Format: {"prompts": [...], "expected_responses": [...]}
-                prompts = data.get("prompts", [])
-                expected_responses = data.get("expected_responses", [])
-                eval_items = [
-                    {"prompt": p, "expected_response": e}
-                    for p, e in zip(prompts, expected_responses)
-                ]
-                return eval_items, None
-        else:
-            raise ValueError("Invalid file format")
-    except SystemExit:
-        raise
-    except Exception as e:
-        emit_structured_log("error", f"Error loading prompts from file: {e}", operation=Operation.LOAD_PROMPTS)
-        sys.exit(1)
-
-def get_interactive_prompts() -> Tuple[List[str], List[str]]:
-    """Get prompts and expected responses interactively."""
-    prompts = []
-    expected_responses = []
-    
-    print("Interactive mode: Enter your prompts and expected responses.")
-    print("Press Enter with empty prompt to finish.")
-    
-    while True:
-        prompt = input(f"\nPrompt {len(prompts) + 1}: ").strip()
-        if not prompt:
-            break
-        
-        expected = input(f"Expected response {len(expected_responses) + 1}: ").strip()
-        
-        prompts.append(prompt)
-        expected_responses.append(expected)
-    
-    if not prompts:
-        print("No prompts entered. Exiting.")
-        sys.exit(1)
-    
-    return prompts, expected_responses
-
-def run_evaluations(args, responses: List[Dict[str, Any]], eval_items: List[Dict],
-                     default_evaluators: Dict[str, Any]) -> list:
-    """Run evaluations against the responses using per-prompt evaluator resolution.
-
-    Args:
-        args: CLI arguments.
-        responses: List of enhanced response dicts (one per prompt, aligned with eval_items by index).
-        eval_items: List of item dicts (prompt, expected_response, evaluators, evaluators_mode).
-        default_evaluators: Resolved default evaluators (from resolve_default_evaluators).
-    """
-    if len(responses) != len(eval_items):
-        raise ValueError(
-            f"Mismatch between number of responses ({len(responses)}) and evaluation items ({len(eval_items)})."
-        )
-
-    model_config = AzureOpenAIModelConfiguration(
-        azure_endpoint=os.environ.get("AZURE_AI_OPENAI_ENDPOINT"),
-        api_key=os.environ.get("AZURE_AI_API_KEY"),
-        api_version=os.environ.get("AZURE_AI_API_VERSION"),
-        azure_deployment=os.environ.get("AZURE_AI_MODEL_NAME"),
-    )
-
-    # Build available context for prerequisite checks
-    has_azure_openai = bool(
-        os.environ.get("AZURE_AI_OPENAI_ENDPOINT")
-        and os.environ.get("AZURE_AI_API_KEY")
-    )
-
-    DEFAULT_PASS_THRESHOLD = 3
-
-    def decorate_metric(metric_id: str, data, threshold: Optional[int] = None):
-        """Augment raw evaluator output with standardized threshold + pass/fail result."""
-        pass_threshold = threshold if threshold is not None else DEFAULT_PASS_THRESHOLD
-        payload = {}
-        if isinstance(data, dict):
-            payload.update(data)
-        else:
-            payload['raw'] = data
-
-        score_val = None
-        if isinstance(data, dict):
-            if metric_id in data:
-                score_val = data[metric_id]
-        if isinstance(score_val, (int, float)):
-            payload['threshold'] = pass_threshold
-            payload['result'] = 'pass' if score_val >= pass_threshold else 'fail'
-        else:
-            payload['threshold'] = pass_threshold
-            payload.setdefault('result', 'unknown')
-        return json.dumps(payload, indent=4)
-
-    # Validate all evaluator names upfront (across defaults and all items)
-    all_evaluator_maps = [default_evaluators]
-    for eval_item in eval_items:
-        if "evaluators" in eval_item:
-            all_evaluator_maps.append(eval_item["evaluators"])
-    for emap in all_evaluator_maps:
-        validate_evaluator_names(emap)
-
-    evaluation_results = []
-    for enhanced_response, eval_item in zip(responses, eval_items):
-        actual_response_text = get_response_text_for_evaluation(enhanced_response)
-        prompt = eval_item.get("prompt", "")
-        expected_response = eval_item.get("expected_response", "")
-        prompt_evaluators = eval_item.get("evaluators")
-        evaluators_mode = eval_item.get("evaluators_mode", "extend")
-
-        # Resolve evaluators for this prompt
-        resolved = resolve_evaluators_for_prompt(
-            prompt_evaluators, evaluators_mode, prompt, default_evaluators,
-        )
-
-        # Build runtime context for prerequisite checks
-        has_tool_defs = bool(
-            args.m365_agent_id and enhanced_response.get("tool_definitions")
-        )
-        available_context = {
-            REQUIRES_AZURE_OPENAI: has_azure_openai,
-            REQUIRES_TOOL_DEFINITIONS: has_tool_defs,
-        }
-
-        results_dict: Dict[str, Optional[str]] = {}
-        evaluators_ran: List[str] = []
-
-        for eval_name, eval_options in resolved.items():
-            # Check prerequisites
-            can_run, warn_msg = check_prerequisites(eval_name, available_context)
-            if not can_run:
-                if warn_msg:
-                    emit_structured_log("warning", f"Evaluator '{eval_name}' prerequisite check failed: {warn_msg}", operation=Operation.EVALUATE)
-                results_dict[eval_name] = None
-                continue
-
-            evaluators_ran.append(eval_name)
-            threshold = get_evaluator_threshold(eval_name, eval_options)
-
-            if eval_name == RELEVANCE:
-                raw_score = RelevanceEvaluator(model_config=model_config)(query=prompt, response=actual_response_text)
-                results_dict[RELEVANCE] = decorate_metric(METRIC_IDS[RELEVANCE], raw_score, threshold)
-            elif eval_name == COHERENCE:
-                raw_score = CoherenceEvaluator(model_config=model_config)(query=prompt, response=actual_response_text)
-                results_dict[COHERENCE] = decorate_metric(METRIC_IDS[COHERENCE], raw_score, threshold)
-            elif eval_name == GROUNDEDNESS:
-                raw_score = GroundednessEvaluator(model_config=model_config)(response=actual_response_text, context=expected_response)
-                results_dict[GROUNDEDNESS] = decorate_metric(METRIC_IDS[GROUNDEDNESS], raw_score, threshold)
-            elif eval_name == TOOL_CALL_ACCURACY:
-                raw_score = ToolCallAccuracyEvaluator(model_config)(
-                    query=prompt,
-                    response=enhanced_response.get("response", actual_response_text),
-                    tool_definitions=enhanced_response["tool_definitions"],
-                )
-                results_dict[TOOL_CALL_ACCURACY] = decorate_metric(METRIC_IDS[TOOL_CALL_ACCURACY], raw_score, threshold)
-            elif eval_name == CITATIONS:
-                fmt_str = eval_options.get("citation_format", "oai_unicode")
-                fmt_map = {
-                    "oai_unicode": CitationFormat.OAI_UNICODE,
-                    "bracket": CitationFormat.LEGACY_BRACKET,
-                    "mixed": CitationFormat.AUTO,
-                }
-                raw_score = CitationsEvaluator(citation_format=fmt_map.get(fmt_str, CitationFormat.OAI_UNICODE))(response=actual_response_text)
-                results_dict[CITATIONS] = decorate_metric(METRIC_IDS[CITATIONS], raw_score, threshold)
-            elif eval_name == EXACT_MATCH:
-                # ExactMatch is binary (match/no-match) — it includes its own result
-                # field, so we skip decorate_metric which assumes a numeric score.
-                case_sensitive = eval_options.get("case_sensitive", False)
-                raw_score = ExactMatchEvaluator(case_sensitive=case_sensitive)(response=actual_response_text, expected_answer=expected_response)
-                results_dict[EXACT_MATCH] = json.dumps(raw_score, indent=4)
-            elif eval_name == PARTIAL_MATCH:
-                case_sensitive = eval_options.get("case_sensitive", False)
-                raw_score = PartialMatchEvaluator(case_sensitive=case_sensitive)(response=actual_response_text, expected_answer=expected_response)
-                results_dict[PARTIAL_MATCH] = decorate_metric(METRIC_IDS[PARTIAL_MATCH], raw_score, threshold)
-
-        evaluation_result = {
-            "prompt": prompt,
-            "response": actual_response_text,
-            "expected_response": expected_response,
-            "evaluators_ran": evaluators_ran,
-            "results": results_dict,
-        }
-
-        # Preserve evaluator config metadata for output
-        if "evaluators" in eval_item:
-            evaluation_result["evaluators"] = eval_item["evaluators"]
-        if "evaluators_mode" in eval_item:
-            evaluation_result["evaluators_mode"] = eval_item["evaluators_mode"]
-
-        if getattr(args, "effective_log_level", "info") == "debug":
-            emit_structured_log(
-                "debug",
-                f"Evaluation completed for prompt='{evaluation_result['prompt']}'. "
-                f"Evaluators: {', '.join(evaluators_ran)}. "
-                f"Scores: {evaluation_result['results']}",
-                operation=Operation.EVALUATE,
-            )
-
-        evaluation_results.append(evaluation_result)
-
-    return evaluation_results
-
-def write_results_to_console(results, agent_name: Optional[str] = None,
-                             agent_id: Optional[str] = None,
-                             cli_version: Optional[str] = None):
-    """Write the response to console."""
-    # ANSI color codes
-    BOLD = '\033[1m'
-    BLUE = '\033[94m'
-    GREEN = '\033[92m'
-    YELLOW = '\033[93m'
-    CYAN = '\033[96m'
-    MAGENTA = '\033[95m'
-    ORANGE = '\033[38;5;208m'
-    RED = '\033[91m'
-    RESET = '\033[0m'
-
-    # Show metadata
-    metadata_parts = []
-    if agent_name:
-        metadata_parts.append(f"Agent Name: {agent_name}")
-    if agent_id:
-        metadata_parts.append(f"Agent ID: {agent_id}")
-    if cli_version:
-        metadata_parts.append(f"CLI Version: {cli_version}")
-    if metadata_parts:
-        print(f"{BOLD}{CYAN}{' | '.join(metadata_parts)}{RESET}")
-        print()
-
-    # Show aggregate statistics if multiple results
-    if len(results) > 1:
-        aggregates = calculate_aggregate_statistics(results)
-        if aggregates:
-            print(f"{BOLD}{BLUE}Aggregate Statistics ({len(results)} prompts):{RESET}")
-            print(f"{BLUE}{'=' * 60}{RESET}")
-
-            for metric_name, stats in aggregates.items():
-                pass_color = GREEN if stats['pass_rate'] >= 80 else YELLOW if stats['pass_rate'] >= 60 else RED
-                prompts_evaluated = stats.get('prompts_evaluated', stats['total_evaluated'])
-                total_prompts = stats.get('total_prompts', len(results))
-                print(f"{BOLD}{CYAN}{metric_name}:{RESET} ({prompts_evaluated}/{total_prompts} prompts)")
-                print(f"  Pass Rate: {pass_color}{stats['pass_rate']:.1f}%{RESET} ({stats['pass_count']}/{stats['total_evaluated']} passed)")
-                print(f"  Avg Score: {MAGENTA}{stats['avg_score']:.2f}{RESET}")
-                if stats.get('threshold') is not None:
-                    print(f"  Threshold: {YELLOW}{stats['threshold']}{RESET}")
-                print()
-
-            print(f"{BLUE}{'=' * 60}{RESET}")
-            print()
-
-    print(f"{BOLD}{BLUE}Individual Results:{RESET}")
-    print(f"{BLUE}{'=' * 50}{RESET}")
-    for i, result in enumerate(results, 1):
-        print(f"{BOLD}{GREEN}Prompt {i}:{RESET} {result['prompt']}")
-
-        # Show which evaluators ran for this prompt
-        evaluators_ran = result.get('evaluators_ran', [])
-        if evaluators_ran:
-            print(f"{BOLD}{CYAN}Evaluators:{RESET} {', '.join(evaluators_ran)}")
-
-        print(f"{BOLD}{CYAN}Response:{RESET} {result['response']}")
-        print(f"{BOLD}{YELLOW}Expected Response:{RESET} {result['expected_response']}")
-
-        # Print metric scores from results
-        metrics = result.get('results', {})
-        for eval_name, v in metrics.items():
-            if v is None:
-                continue  # Skip null/N/A scores from skipped evaluators
-            display_name = pascal_case_to_title(eval_name)
-            if eval_name == RELEVANCE:
-                color = MAGENTA
-            elif eval_name == COHERENCE:
-                color = ORANGE
-            else:
-                color = BLUE
-            print(f"{BOLD}{color}{display_name}:{RESET} {v}")
-        print(f"{BLUE}{'-' * 30}{RESET}")
-
-def extract_eval_score(data: dict, metric_id: str) -> Optional[Dict]:
-    """Extract an EvalScore object from a decorated metric dict.
-
-    Maps internal decorated-metric format to schema EvalScore:
-    {score, result, threshold} (required) + reason, evaluator (optional).
-    """
-    DEFAULT_THRESHOLD = 3  # fallback; decorate_metric should always set this
-
-    score_val = None
-    if metric_id in data and isinstance(data[metric_id], (int, float)):
-        score_val = data[metric_id]
-    if score_val is None:
-        return None
-
-    result = data.get("result")
-    if result not in ("pass", "fail"):
-        result = "pass" if score_val >= data.get("threshold", DEFAULT_THRESHOLD) else "fail"
-
-    eval_score: Dict[str, Any] = {
-        "score": score_val,
-        "result": result,
-        "threshold": data.get("threshold", DEFAULT_THRESHOLD),
-    }
-    reason = data.get(f"{metric_id}_reason") or data.get("reason")
-    if reason:
-        eval_score["reason"] = reason
-    return eval_score
-
-
-def convert_result_to_eval_item(result: Dict) -> Dict:
-    """Convert an internal evaluation result dict to a schema-compliant EvalItem.
-
-    Internal format (from run_evaluations):
-        {prompt, response, expected_response, results: {Relevance: "JSON", ...},
-         evaluators_ran: [...], evaluators: {...}, evaluators_mode: "..."}
-    Schema EvalItem format:
-        {prompt, response, expected_response, scores: {relevance: EvalScore, ...},
-         evaluators: {...}, evaluators_mode: "..."}
-    """
-    item: Dict[str, Any] = {
-        "prompt": result["prompt"],
-        "response": result["response"],
-        "expected_response": result["expected_response"],
-    }
-
-    # Preserve evaluator config in output
-    if "evaluators" in result:
-        item["evaluators"] = result["evaluators"]
-    if "evaluators_mode" in result:
-        item["evaluators_mode"] = result["evaluators_mode"]
-
-    scores: Dict[str, Any] = {}
-    results_dict = result.get("results", {})
-
-    # EvalScore metrics (all share the same schema shape: {score, result, threshold})
-    for eval_key, schema_key in [
-        (RELEVANCE, "relevance"),
-        (COHERENCE, "coherence"),
-        (GROUNDEDNESS, "groundedness"),
-        (TOOL_CALL_ACCURACY, "toolCallAccuracy"),
-    ]:
-        raw = results_dict.get(eval_key)
-        if not raw:
-            continue
-        data = json.loads(raw) if isinstance(raw, str) else raw
-        eval_score = extract_eval_score(data, METRIC_IDS[eval_key])
-        if eval_score:
-            scores[schema_key] = eval_score
-
-    # Citations → CitationScore
-    raw_citations = results_dict.get(CITATIONS)
-    if raw_citations:
-        data = json.loads(raw_citations) if isinstance(raw_citations, str) else raw_citations
-        count = data.get("citations", 0)
-        cit_result = data.get("result")
-        if cit_result not in ("pass", "fail"):
-            cit_result = "pass" if count >= data.get("threshold", 1) else "fail"
-
-        citation_score: Dict[str, Any] = {
-            "count": count,
-            "result": cit_result,
-            "threshold": data.get("threshold", 1),
-        }
-        if "citation_format" in data:
-            citation_score["format"] = data["citation_format"]
-        scores["citations"] = citation_score
-
-    # ExactMatch → ExactMatchScore
-    raw_exact = results_dict.get(EXACT_MATCH)
-    if raw_exact:
-        data = json.loads(raw_exact) if isinstance(raw_exact, str) else raw_exact
-        is_match = data.get("exact_match", 0.0) == 1.0
-        scores["exactMatch"] = {
-            "match": is_match,
-            "result": data.get("result", "pass" if is_match else "fail"),
-            "reason": data.get("exact_match_reason", ""),
-        }
-
-    # PartialMatch → PartialMatchScore
-    raw_partial = results_dict.get(PARTIAL_MATCH)
-    if raw_partial:
-        data = json.loads(raw_partial) if isinstance(raw_partial, str) else raw_partial
-        scores["partialMatch"] = {
-            "score": data.get("partial_match", 0.0),
-            "result": data.get("result", "fail"),
-            "threshold": data.get("threshold", 0.5),
-            "reason": data.get("partial_match_reason", ""),
-        }
-
-    if scores:
-        item["scores"] = scores
-
-    return item
-
-
-def write_results_to_json(results: List[Dict], output_file: str, agent_id: Optional[str] = None,
-                          default_evaluators: Optional[Dict[str, Any]] = None,
-                          agent_name: Optional[str] = None,
-                          cli_version: Optional[str] = None):
-    """Write results to a schema-compliant eval document JSON file.
-
-    Output follows the eval-document.schema.json format:
-    {schemaVersion, metadata, default_evaluators?, items: [EvalItem]}
-    """
-    try:
-        try:
-            current_version = SchemaVersionManager().get_current_version()
-        except Exception:
-            current_version = "1.0.0"
-
-        items = [convert_result_to_eval_item(r) for r in results]
-
-        metadata: Dict[str, Any] = {
-            "evaluatedAt": datetime.now(timezone.utc).isoformat(),
-        }
-        if agent_id:
-            metadata["agentId"] = agent_id
-        if agent_name:
-            metadata["agentName"] = agent_name
-        if cli_version:
-            metadata["cliVersion"] = cli_version
-
-        output_data: Dict[str, Any] = {
-            "schemaVersion": current_version,
-            "metadata": metadata,
-        }
-
-        if default_evaluators is not None:
-            output_data["default_evaluators"] = default_evaluators
-
-        output_data["items"] = items
-
-        with open(output_file, 'w', encoding='utf-8') as f:
-            json.dump(output_data, f, indent=2, ensure_ascii=False)
-        emit_structured_log("info", f"Results saved to {output_file}", operation=Operation.WRITE_OUTPUT)
-    except Exception as e:
-        emit_structured_log("error", f"Error writing to JSON file: {e}", operation=Operation.WRITE_OUTPUT)
-        sys.exit(1)
-
-def write_results_to_csv(results: List[Dict], output_file: str,
-                         agent_name: Optional[str] = None, agent_id: Optional[str] = None,
-                         cli_version: Optional[str] = None):
-    """Write results to CSV file."""
-    try:
-        with open(output_file, 'w', newline='', encoding='utf-8') as f:
-            if results:
-                # Write metadata header
-                metadata_parts = []
-                if agent_name:
-                    metadata_parts.append(f"Agent Name: {agent_name}")
-                if agent_id:
-                    metadata_parts.append(f"Agent ID: {agent_id}")
-                if cli_version:
-                    metadata_parts.append(f"CLI Version: {cli_version}")
-                if metadata_parts:
-                    f.write(f"# {' | '.join(metadata_parts)}\n")
-
-                # Write aggregate statistics first if multiple results
-                if len(results) > 1:
-                    aggregates = calculate_aggregate_statistics(results)
-                    if aggregates:
-                        f.write("# AGGREGATE STATISTICS\n")
-                        f.write("Metric,Prompts Evaluated,Total Prompts,Pass Rate (%),Passed,Failed,Avg Score,Threshold\n")
-                        for metric_name, stats in aggregates.items():
-                            threshold_str = str(stats.get('threshold', 'N/A'))
-                            prompts_evaluated = stats.get('prompts_evaluated', stats['total_evaluated'])
-                            total_prompts = stats.get('total_prompts', len(results))
-                            f.write(f"{metric_name},{prompts_evaluated},{total_prompts},{stats['pass_rate']:.1f},{stats['pass_count']},{stats['fail_count']},{stats['avg_score']:.2f},{threshold_str}\n")
-                        f.write("\n# INDIVIDUAL RESULTS\n")
-
-                # Write individual results (exclude internal fields)
-                exclude_keys = {'evaluators_ran', 'evaluators', 'evaluators_mode'}
-                fieldnames = [k for k in results[0].keys() if k not in exclude_keys]
-                writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
-                writer.writeheader()
-                writer.writerows(results)
-            emit_structured_log("info", f"Results saved to {output_file}", operation=Operation.WRITE_OUTPUT)
-    except Exception as e:
-        emit_structured_log("error", f"Error writing to CSV file: {e}", operation=Operation.WRITE_OUTPUT)
-        sys.exit(1)
-
-def parse_arguments():
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="M365 Copilot Agent Evaluation CLI",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Run with default prompts
-  python main.py
-
-  # Run with custom prompts
-  python main.py --prompts "What is Microsoft Graph?" --expected "Microsoft Graph is a gateway..."
-
-  # Run with prompts from file
-  python main.py --prompts-file prompts.json
-
-  # Interactive mode
-  python main.py --interactive
-
-  # Save results to JSON
-  python main.py --output results.json
-
-  # Save results to CSV
-  python main.py --output results.csv
-
-  # Save results to HTML and open in browser
-  python main.py --output report.html
-
-  # Debug-level diagnostics
-  python main.py --log-level debug
-
-  # Sign out and clear cached authentication tokens
-  python main.py --signout
-        """
-    )
-    
-    # Input options (mutually exclusive)
-    input_group = parser.add_mutually_exclusive_group()
-    input_group.add_argument(
-        '--prompts', 
-        nargs='+', 
-        help='List of prompts to evaluate'
-    )
-    input_group.add_argument(
-        '--prompts-file', 
-        type=str, 
-        help='JSON file containing prompts and expected responses'
-    )
-    input_group.add_argument(
-        '--interactive', 
-        action='store_true', 
-        help='Interactive mode to enter prompts'
-    )
-    
-    # Expected responses (only used with --prompts)
-    parser.add_argument(
-        '--expected', 
-        nargs='+', 
-        help='List of expected responses (must match number of prompts)'
-    )
-    
-    # Agent ID (--m365-agent-id is primary, --agent-id kept for backward compatibility)
-    parser.add_argument(
-        '--m365-agent-id', '--agent-id',
-        type=str,
-        default=os.environ.get("M365_AGENT_ID") or os.environ.get("AGENT_ID"),
-        help='Agent ID (default from M365_AGENT_ID environment variable)'
-    )
-
-    # Output options
-    parser.add_argument(
-        '--output', 
-        type=str, 
-        help='Output file path. Format is determined by file extension: .json, .csv, .html. If not provided, results are printed to console.'
-    )
-    
-    # Behavior options
-    parser.add_argument(
-        '--log-level',
-        nargs='?',
-        const='info',
-        action='append',
-        help='Set log verbosity: debug, info, warning, error. Bare --log-level resolves to info.'
-    )
-
-    parser.add_argument(
-        '--signout',
-        action='store_true',
-        help='Sign out and clear cached authentication tokens'
-    )
-    
-    return parser.parse_args()
-
-def validate_environment() -> CallPath:
-    """Validate required environment variables."""
-    required_env_vars = [
-        "AZURE_AI_OPENAI_ENDPOINT",
-        "AZURE_AI_API_KEY",
-        "AZURE_AI_API_VERSION",
-        "AZURE_AI_MODEL_NAME",
-        # Chat API specific
-        "COPILOT_API_ENDPOINT",
-        "X_SCENARIO_HEADER"
-    ]
-
-    if os.environ.get("COPILOT_API_ACCESS_TOKEN"):
-        call_path = CallPath.ACCESS_TOKEN
-        required_env_vars.append("COPILOT_API_ACCESS_TOKEN")
-    else:
-        call_path = CallPath.COPILOT_AUTH
-        required_env_vars.extend([
-            "M365_EVAL_CLIENT_ID",
-            "TENANT_ID"
-        ])
-    
-    missing_vars = [var for var in required_env_vars if not os.environ.get(var)]
-    if missing_vars:
-        emit_structured_log(
-            "error",
-            "Missing required environment variables: "
-            f"{', '.join(missing_vars)}. Please ensure your .env file contains "
-            "all required Azure configuration.",
-            operation=Operation.VALIDATE_ENV,
-        )
-        sys.exit(1)
-    return call_path
-
-def validate_endpoint_url(url: str, allowed_domains: List[str]) -> bool:
-    """Validate URL against security requirements."""
-    try:
-        parsed = urllib.parse.urlparse(url)
-        
-        # Check for dangerous schemes
-        if parsed.scheme in ['javascript', 'data']:
-            raise ValueError(f"Dangerous URL scheme detected: {parsed.scheme}")
-        
-        # Check for HTTPS requirement
-        if parsed.scheme != 'https':
-            raise ValueError(f"Only HTTPS URLs are allowed, got: {parsed.scheme}")
-        
-        # Check if domain is in allowed list
-        if parsed.netloc not in allowed_domains:
-            raise ValueError(f"Domain not in allowed list: {parsed.netloc}")
-        
-        # Reject fragment URLs
-        if parsed.fragment:
-            raise ValueError("Fragment URLs are not allowed")
-        
-        return True
-        
-    except ValueError:
-        # Re-raise ValueError exceptions
-        raise
-    except Exception as e:
-        # Convert other parsing errors to ValueError
-        raise ValueError(f"Invalid URL format: {url}") from e
-
-def get_prompt_datasets(args) -> Tuple[List[Dict], Optional[Dict]]:
-    """Get prompts and expected responses based on command line arguments.
-
-    Returns:
-        Tuple of (eval_items, default_evaluators).
-    """
-    if args.prompts:
-        if args.expected and len(args.prompts) != len(args.expected):
-            emit_structured_log(
-                "error",
-                "Number of prompts must match number of expected responses. "
-                "Update --expected values to match the prompt count.",
-            )
-            sys.exit(1)
-        expected_responses = args.expected or [""] * len(args.prompts)
-        eval_items = [
-            {"prompt": p, "expected_response": e}
-            for p, e in zip(args.prompts, expected_responses)
-        ]
-        return eval_items, None
-    elif args.prompts_file:
-        return load_prompts_from_file(args.prompts_file)
-    elif args.interactive:
-        prompts, expected_responses = get_interactive_prompts()
-        eval_items = [
-            {"prompt": p, "expected_response": e}
-            for p, e in zip(prompts, expected_responses)
-        ]
-        return eval_items, None
-    else:
-        prompts, expected_responses = get_default_prompts_and_responses()
-        eval_items = [
-            {"prompt": p, "expected_response": e}
-            for p, e in zip(prompts, expected_responses)
-        ]
-        return eval_items, None
-
-def fetch_available_agents(copilot_api_endpoint: str, access_token: str, user_oid: str) -> List[Dict[str, Any]]:
-    """
-    Fetch available agents for the user from the Copilot API.
-    
-    Args:
-        access_token: Bearer token for API authentication
-        user_oid: User object ID for agent filtering
-        
-    Returns:
-        List of agent dictionaries.
-    """
-    request_headers = {
-        "Content-Type": "application/json",
-        "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
-        "Authorization": f"Bearer {access_token}"
-    }
-    
-    try:
-        # Build the query parameter with participant info
-        request_data = json.dumps({"participant": {"id": user_oid}})
-        query_param = urllib.parse.quote(request_data)
-        
-        # Try to fetch agents from /GetGptList endpoint
-        req = urllib.request.Request(
-            f"{copilot_api_endpoint}/GetGptList?request={query_param}",
-            headers=request_headers,
-            method="GET"
-        )
-        with urllib.request.urlopen(req, timeout=120) as resp:
-            data = json.loads(resp.read().decode("utf-8"))
-            agents = data.get("gptList", [])
-            return agents
-    except urllib.error.HTTPError as e:
-        # If endpoint doesn't exist or returns error, return empty list
-        emit_structured_log("warning", f"Unable to fetch agents list (HTTP {e.code}).", operation=Operation.FETCH_AGENTS)
-        return []
-    except Exception as e:
-        emit_structured_log("warning", f"Error fetching agents: {e}", operation=Operation.FETCH_AGENTS)
-        return []
-
-def select_agent_interactively(agents: List[Dict[str, Any]]) -> Tuple[Optional[str], Optional[str]]:
-    """
-    Display an interactive agent selector using questionary.
-
-    Args:
-        agents: List of agent dictionaries.
-
-    Returns:
-        Tuple of (agent_id, agent_name) or (None, None) if cancelled/skipped
-    """
-    if not agents:
-        return None, None
-
-    # Build id→name lookup and choices
-    id_to_name: Dict[str, str] = {}
-    choices = []
-    sorted_agents = sorted(agents, key=lambda x: (not x.get('isOwner', False), x.get('name', '')))
-    for agent in sorted_agents:
-        agent_id = agent.get("gptId", "Unknown")
-        agent_name = agent.get("name", "Unknown")
-        agent_description = agent.get("description", "Unknown")
-        agent_is_owner = agent.get('isOwner')
-        id_to_name[agent_id] = agent_name
-
-        # Format the display text
-        display_text = f"{agent_name} ({agent_id}, IsOwner: {agent_is_owner}) - {agent_description}"
-
-        choices.append(questionary.Choice(title=display_text, value=agent_id))
-
-    # Display the selection prompt
-    selected_agent = questionary.select(
-        "Select an agent to evaluate:",
-        choices=choices,
-        use_shortcuts=True,
-        use_arrow_keys=True
-    ).ask()
-
-    return selected_agent, id_to_name.get(selected_agent) if selected_agent else None
-
-@functools.lru_cache(maxsize=1)
-def _get_iana_timezone_name() -> str:
-    """Get the IANA timezone name from the system using tzlocal.
-
-    Tries get_localzone_name() first; falls back to str(get_localzone()) when the
-    former raises (e.g. no zone configured on some Unix systems).  Result is cached
-    after the first call so tzlocal is only invoked once per session.
-    """
-    try:
-        return tzlocal.get_localzone_name()
-    except Exception:
-        return str(tzlocal.get_localzone())
-
-
-@functools.lru_cache(maxsize=1)
-def _get_location_info() -> Dict[str, Any]:
-    """Return a locationInfo dict containing the local UTC offset and IANA timezone name.
-
-    Result is cached after the first call so the computation runs only once per session.
-    """
-    now = datetime.now().astimezone()
-    utc_offset = now.utcoffset()
-    offset_hours = int(utc_offset.total_seconds() // 3600) if utc_offset is not None else 0
-    return {
-        "timeZoneOffset": offset_hours,
-        "timeZone": _get_iana_timezone_name(),
-    }
-
-
-def build_chat_payload(prompt: str, user_oid: str, agent_id: str|None) -> bytes:
-    message = {
-        "message": {
-            "text": prompt,
-            "author": "user",
-            "messageType": "chat",
-            "timestamp": datetime.now(timezone.utc).isoformat(),
-            "locationInfo": _get_location_info(),
-            "from": {
-                    "id": user_oid,
-            }
-        },
-        "verbosity": "verbose", # To enable detailed telemetry in response (to extract tool usage, etc.)
-    }
-    
-    if agent_id:
-            message["gpts"] = [
-                {
-                        "id": agent_id.strip(),
-                        "source": "MOS3"
-                }
-            ]
-            message["optionsSets"] = [
-                "disable_action_confirmation"   # Disable 3P action confirmation prompts for agents while scraping
-            ]
-  
-    return json.dumps(message).encode("utf-8")
-
-def send_prompt_to_agent_in_sydney(prompts: List[str], copilot_api_endpoint: str, access_token: str, user_oid: str, args) -> List[Dict[str, Any]]:
-    """ Send prompts to the chat API and return enhanced responses. """
-
-    request_headers = {
-        "Content-Type": "application/json",
-        "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
-        "Authorization": f"Bearer {access_token}"
-        }
-    raw_responses: List[Tuple[str, str]] = []
-    for i, prompt in enumerate(prompts, 1):
-        if getattr(args, "effective_log_level", "info") in ("info", "debug"):
-            emit_structured_log("info", f"Processing prompt {i}/{len(prompts)}.", operation=Operation.SEND_PROMPT)
-
-        # Build the payload
-        payload = build_chat_payload(prompt, user_oid, args.m365_agent_id)
-        if getattr(args, "effective_log_level", "info") == "debug":
-            emit_structured_log("debug", f"[Sydney] Sending payload: {payload.decode('utf-8')}", operation=Operation.SEND_PROMPT)
-
-        # Send the request to /chat
-        req = urllib.request.Request(f"{copilot_api_endpoint}/chat", data=payload, headers=request_headers, method="POST")
-        try:
-            with urllib.request.urlopen(req, timeout=120) as resp:
-                raw = resp.read().decode("utf-8", errors="replace")
-        except urllib.error.HTTPError as e:
-            error_body = None
-            try:
-                error_body = e.read().decode("utf-8", errors="replace")
-            except Exception:
-                pass
-            msg = f"Chat API request failed (HTTP {e.code} {e.reason})."
-            if error_body:
-                msg += f" Body: {error_body[:500]}"
-            raise RuntimeError(msg) from e
-        except urllib.error.URLError as e:
-            raise RuntimeError(f"Chat API connection error: {getattr(e, 'reason', str(e))}") from e
-
-        if getattr(args, "effective_log_level", "info") == "debug":
-            emit_structured_log("debug", f"[Sydney] Raw response: {raw}", operation=Operation.SEND_PROMPT)
-
-        # Store raw response for enhancement
-        raw_responses.append((prompt, raw.strip()))
-
-    # Extract enhanced responses using the new extractor
-    enhanced_responses = extract_enhanced_responses(raw_responses, log_level=getattr(args, "effective_log_level", "info"))
-
-    if getattr(args, "effective_log_level", "info") == "debug":
-        for idx, enhanced in enumerate(enhanced_responses, 1):
-            metadata = enhanced.get("metadata", {})
-            context = {
-                "request-id": metadata.get("request_id"),
-                "conversation-id": metadata.get("conversation_id"),
-                "message-id": metadata.get("message_id"),
-                "operation": Operation.SEND_PROMPT,
-            }
-            entry = format_structured_log_entry(
-                level="debug",
-                message=f"Response IDs for prompt {idx}/{len(enhanced_responses)}.",
-                logger_name=CLI_LOGGER_NAME,
-                run_context=context,
-            )
-            DIAGNOSTIC_RECORDS.append(entry)
-            CLI_LOGGER.log(logging.DEBUG, render_diagnostic(entry))
+from cli_args import parse_arguments, should_bypass_min_version_check
+from env_validator import (
+    ALLOWED_ENDPOINTS,
+    validate_endpoint_url,
+    validate_environment,
+)
+from common import (
+    ENV_AZURE_AI_OPENAI_ENDPOINT,
+    ENV_AZURE_AI_API_KEY,
+    ENV_AZURE_AI_API_VERSION,
+    ENV_AZURE_AI_MODEL_NAME,
+    ENV_TENANT_ID,
+    ENV_WORK_IQ_A2A_ENDPOINT,
+    ENV_WORK_IQ_A2A_CLIENT_ID,
+    ENV_WORK_IQ_A2A_SCOPES,
+    RunConfig,
+)
+from prompt_loader import get_prompt_datasets
+from agent_selector import select_agent_interactively
+from evaluation_runner import PipelineConfig, run_pipeline
+from result_writer import output_results
 
-    return enhanced_responses
+from dataclasses import replace
 
-def output_results(results: List[Dict], args, default_evaluators: Optional[Dict[str, Any]] = None,
-                    agent_name: Optional[str] = None, cli_version: Optional[str] = None):
-    """Output results based on specified format."""
-    metadata_kwargs = dict(
-        agent_name=agent_name,
-        agent_id=getattr(args, 'm365_agent_id', None),
-        cli_version=cli_version,
-    )
-    if args.output:
-        output_lower = args.output.lower()
-        if output_lower.endswith('.json'):
-            write_results_to_json(results, args.output, default_evaluators=default_evaluators,
-                                  **metadata_kwargs)
-        elif output_lower.endswith('.csv'):
-            write_results_to_csv(results, args.output, **metadata_kwargs)
-        elif output_lower.endswith('.html'):
-            write_results_to_html(results, args.output, **metadata_kwargs)
-            abs_path = os.path.abspath(args.output)
-            webbrowser.open(f'file://{abs_path}')
-        else:
-            write_results_to_json(results, args.output, default_evaluators=default_evaluators,
-                                  **metadata_kwargs)
-    else:
-        write_results_to_console(results, **metadata_kwargs)
 
 def main():
     """Main function to orchestrate the evaluation process."""
@@ -1107,136 +67,159 @@ def main():
         )
         sys.exit(2)
 
-    args.effective_log_level = effective_log_level
-    configure_cli_logging(effective_log_level)
-    emit_structured_log("info", f"Log level set to '{effective_log_level}'.", operation=Operation.SETUP)
+    config = replace(
+        RunConfig.from_namespace(args),
+        effective_log_level=effective_log_level,
+    )
+    configure_cli_logging(config.effective_log_level)
+    emit_structured_log("info", f"Log level set to '{config.effective_log_level}'.", operation=Operation.SETUP)
 
     # Check minimum version before proceeding
-    quiet_for_version = effective_log_level in ("warning", "error")
+    quiet_for_version = config.effective_log_level in ("warning", "error")
     cli_version = get_cli_version(quiet=quiet_for_version)
-    if not should_bypass_min_version_check(args) and not check_min_version(cli_version, quiet=quiet_for_version):
+    if not should_bypass_min_version_check(config) and not check_min_version(cli_version, quiet=quiet_for_version):
         sys.exit(1)
 
     # Validate environment variables required for evaluation
-    call_path = validate_environment()
-    copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
-    validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
-    
-    user_oid = ""
-
-    if call_path == CallPath.ACCESS_TOKEN:
-        access_token = os.environ["COPILOT_API_ACCESS_TOKEN"]
-    else:
-      scopes_str = os.environ.get(
-          "COPILOT_SCOPES", "https://substrate.office.com/sydney/.default"
-          )
-          
-      auth_handler = AuthHandler(
-          client_id=os.environ["M365_EVAL_CLIENT_ID"],
-          tenant_id=os.environ["TENANT_ID"],
-          scopes_str=scopes_str
-          )
+    validate_environment()
 
-      # Signout user
-      if args.signout:
-          try:
-              auth_handler.clear_cache()
-          except Exception as e:
-              emit_structured_log("error", f"Error during signout: {e}", operation=Operation.AUTHENTICATE)
-              sys.exit(1)
-          sys.exit(0)
+    a2a_endpoint = os.environ[ENV_WORK_IQ_A2A_ENDPOINT]
+    validate_endpoint_url(a2a_endpoint, ALLOWED_ENDPOINTS)
 
-      # Authenticate before loading prompts
-      try:
-        auth_result = auth_handler.acquire_token_interactive() or {}
-        access_token = auth_result.get("access_token") or ""
-        if not access_token:
-            raise RuntimeError("Failed to acquire access token from authentication result")
-
-        id_token_claims = auth_result.get("id_token_claims")
-        if not isinstance(id_token_claims, dict):
+    a2a_scopes_str = os.environ.get(ENV_WORK_IQ_A2A_SCOPES, "")
+    a2a_auth_handler = AuthHandler(
+        client_id=os.environ[ENV_WORK_IQ_A2A_CLIENT_ID],
+        tenant_id=os.environ[ENV_TENANT_ID],
+        scopes_str=a2a_scopes_str,
+    )
+    if config.signout:
+        try:
+            a2a_auth_handler.clear_cache()
+        except Exception as e:
             emit_structured_log(
-                "warning", "id_token_claims is missing or invalid in authentication result",
+                "error",
+                f"Error during signout: {e}",
                 operation=Operation.AUTHENTICATE,
             )
-        else:
-            user_oid = id_token_claims.get("oid") or ""
-        
-      except Exception as e:
-          emit_structured_log("error", f"Error during authentication: {e}", operation=Operation.AUTHENTICATE)
-          if effective_log_level == "debug":
-              import traceback
-              traceback.print_exc()
-          sys.exit(1)
+            sys.exit(1)
+        sys.exit(0)
 
-    if not user_oid and access_token:
-        # Fallback: extract from access token.
-        user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
+    try:
+        a2a_auth_result = a2a_auth_handler.acquire_token_interactive() or {}
+        a2a_access_token = a2a_auth_result.get("access_token") or ""
+        if not a2a_access_token:
+            raise RuntimeError("Failed to acquire A2A access token")
+    except Exception as e:
+        emit_structured_log(
+            "error",
+            f"Error during A2A authentication: {e}",
+            operation=Operation.AUTHENTICATE,
+        )
+        if config.effective_log_level == "debug":
+            traceback.print_exc()
+        sys.exit(1)
+    try:
+        agent_client = A2AClient(
+            a2a_endpoint=a2a_endpoint,
+            access_token=a2a_access_token,
+            logger=CLI_LOGGER,
+            diagnostic_records=DIAGNOSTIC_RECORDS,
+        )
+    except Exception as e:
+        emit_structured_log(
+            "error",
+            f"Failed to initialize A2A client: {e}",
+            operation=Operation.SETUP,
+        )
+        sys.exit(1)
 
     # 1. Load evaluation datasets
-    eval_items, file_default_evaluators = get_prompt_datasets(args)
+    eval_items, file_default_evaluators = get_prompt_datasets(config)
     default_evaluators = resolve_default_evaluators(file_default_evaluators)
-    prompts = [eval_item.get("prompt", "") for eval_item in eval_items]
 
-    if effective_log_level in ("info", "debug"):
-        emit_structured_log("info", f"Running evaluation on {len(prompts)} prompt(s).", operation=Operation.SETUP)
+    if config.effective_log_level in ("info", "debug"):
+        multi_turn_count = sum(1 for item in eval_items if "turns" in item)
+        single_turn_count = len(eval_items) - multi_turn_count
+        emit_structured_log(
+            "info",
+            f"Running evaluation on {len(eval_items)} item(s) "
+            f"({single_turn_count} single-turn, {multi_turn_count} multi-turn).",
+            operation=Operation.SETUP,
+        )
 
     agent_name = None
     try:
-        # 3. Agent selection - if no agent ID provided, prompt user to select
-        if not args.m365_agent_id:
-            if effective_log_level in ("info", "debug"):
+        # 2. Agent selection - when no agent ID is provided, discover agents
+        # via the active client (A2A) and prompt interactively.
+        if not config.m365_agent_id:
+            if config.effective_log_level in ("info", "debug"):
                 emit_structured_log("info", "No agent ID provided. Fetching available agents.", operation=Operation.FETCH_AGENTS)
-            
-            available_agents = fetch_available_agents(copilot_api_endpoint, access_token, user_oid)
+
+            available_agents = agent_client.fetch_available_agents()
             if not available_agents:
                 emit_structured_log(
-                  "error",
-                  "No agents are available for interactive selection. Re-run with "
-                  "--m365-agent-id or set M365_AGENT_ID.",
-                  operation=Operation.FETCH_AGENTS,
+                    "error",
+                    "No agents are available for interactive selection."
+                    " Re-run with --m365-agent-id or set M365_AGENT_ID.",
+                    operation=Operation.FETCH_AGENTS,
                 )
                 sys.exit(1)
 
             selected_agent_id, agent_name = select_agent_interactively(available_agents)
             if selected_agent_id:
-                args.m365_agent_id = selected_agent_id
-                if effective_log_level in ("info", "debug"):
-                    emit_structured_log("info", f"Selected agent: {args.m365_agent_id}", operation=Operation.FETCH_AGENTS)
+                config = replace(config, m365_agent_id=selected_agent_id)
+                if config.effective_log_level in ("info", "debug"):
+                    emit_structured_log("info", f"Selected agent: {config.m365_agent_id}", operation=Operation.FETCH_AGENTS)
             else:
                 emit_structured_log(
-                        "error",
-                        "No agent selected. Re-run with --m365-agent-id or set M365_AGENT_ID.",
-                        operation=Operation.FETCH_AGENTS,
+                    "error",
+                    "No agent selected. Re-run with --m365-agent-id or set M365_AGENT_ID.",
+                    operation=Operation.FETCH_AGENTS,
                 )
                 sys.exit(1)
-
-        # 4. Send prompts to chat API
-        responses = send_prompt_to_agent_in_sydney(prompts, copilot_api_endpoint, access_token, user_oid, args)
     except Exception as e:
-        emit_structured_log("error", f"Error sending prompts to chat API: {e}", operation=Operation.SEND_PROMPT)
-        if effective_log_level == "debug":
-            import traceback
+        emit_structured_log("error", f"Error during agent discovery: {e}", operation=Operation.FETCH_AGENTS)
+        if config.effective_log_level == "debug":
             traceback.print_exc()
         sys.exit(1)
 
+    # Pre-resolve agent endpoint (A2A agent card lookup)
+    if config.m365_agent_id:
+        agent_client.resolve_agent(config.m365_agent_id)
+
+    # 3. Build pipeline config and run evaluation pipeline
+    model_config = AzureOpenAIModelConfiguration(
+        azure_endpoint=os.environ.get(ENV_AZURE_AI_OPENAI_ENDPOINT),
+        api_key=os.environ.get(ENV_AZURE_AI_API_KEY),
+        api_version=os.environ.get(ENV_AZURE_AI_API_VERSION),
+        azure_deployment=os.environ.get(ENV_AZURE_AI_MODEL_NAME),
+    )
+    has_azure_openai = bool(
+        os.environ.get(ENV_AZURE_AI_OPENAI_ENDPOINT)
+        and os.environ.get(ENV_AZURE_AI_API_KEY)
+    )
+
+    pipeline = PipelineConfig(
+        agent_client=agent_client,
+        model_config=model_config,
+        has_azure_openai=has_azure_openai,
+        default_evaluators=default_evaluators,
+    )
+
+    results = run_pipeline(pipeline, eval_items, config)
+
+    # 4. Output results
+    output_results(
+        results, config, default_evaluators=default_evaluators,
+        agent_name=agent_name, cli_version=str(cli_version) if cli_version else None)
     
-    # 5. Run evaluations
-    if effective_log_level in ("info", "debug"):
-        emit_structured_log("info", "Running evaluations.", operation=Operation.EVALUATE)
-    results = run_evaluations(args, responses, eval_items, default_evaluators)
-    
-    # 6. Output results
-    output_results(results, args, default_evaluators=default_evaluators,
-                   agent_name=agent_name, cli_version=str(cli_version) if cli_version else None)
-    
-    if effective_log_level in ("info", "debug"):
+    if config.effective_log_level in ("info", "debug"):
         emit_structured_log(
             "info",
-            f"Evaluation completed successfully. Processed {len(prompts)} prompt(s).",
+            f"Evaluation completed successfully. Processed {len(eval_items)} item(s).",
             operation=Operation.EVALUATE,
         )
 
 # Call the main function when script is run directly
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     main()