@microsoft/m365-copilot-eval 1.4.0-preview.1 → 1.6.0-preview.1

package/src/clients/cli/main.py

@@ -1,1546 +1,58 @@
-import json
+"""M365 Copilot Agent Evaluation CLI — thin orchestrator.
+
+Delegates to focused modules:
+  cli_args          – argument parsing & version-check bypass
+  env_validator     – environment validation & URL security
+  prompt_loader     – dataset loading & agent selection
+  evaluation_runner – pipeline, evaluator dispatch, retry
+  result_writer     – console / JSON / CSV / HTML output
+"""
+
 import os
-import argparse
 import sys
-import csv
-import logging
-import time
-import webbrowser
-import urllib.parse
-import questionary
-from dataclasses import dataclass, field
-from enum import Enum
-from typing import List, Dict, Tuple, Optional, Any
-
-from api_clients.A2A import A2AClient
-from api_clients.REST import SydneyClient
-from api_clients.base_agent_client import BaseAgentClient
+import traceback
 
-from azure.ai.evaluation import (
-    AzureOpenAIModelConfiguration,
-    RelevanceEvaluator,
-    CoherenceEvaluator,
-    GroundednessEvaluator,
-    ToolCallAccuracyEvaluator
-)
+from azure.ai.evaluation import AzureOpenAIModelConfiguration
 from dotenv import load_dotenv
-from auth.auth_handler import AuthHandler
-from custom_evaluators.CitationsEvaluator import CitationsEvaluator, CitationFormat
-from custom_evaluators.ExactMatchEvaluator import ExactMatchEvaluator
-from custom_evaluators.PartialMatchEvaluator import PartialMatchEvaluator
-from generate_report import generate_html_report, calculate_aggregate_statistics
-from response_extractor import get_response_text_for_evaluation
-from schema_handler import DocumentUpgrader, SchemaVersionManager
-from common import (
-    RELEVANCE,
-    COHERENCE,
-    GROUNDEDNESS,
-    TOOL_CALL_ACCURACY,
-    CITATIONS,
-    EXACT_MATCH,
-    PARTIAL_MATCH,
-    REQUIRES_AZURE_OPENAI,
-    REQUIRES_TOOL_DEFINITIONS,
-    METRIC_IDS,
-    STATUS_PASS,
-    STATUS_FAIL,
-    STATUS_ERROR,
-    STATUS_PARTIAL,
-    STATUS_UNKNOWN,
-    pascal_case_to_title,
-)
-from evaluator_resolver import (
-    validate_evaluator_names,
-    check_prerequisites,
-    resolve_default_evaluators,
-    resolve_evaluators_for_prompt,
-    get_evaluator_threshold,
-)
-from version_check import check_min_version, get_cli_version
-from datetime import datetime, timezone
-from pathlib import Path
 
+from api_clients.A2A import A2AClient
+from auth.auth_handler import AuthHandler, make_token_refresh_fn
+from evaluator_resolver import resolve_default_evaluators
+from version_check import check_min_version, get_cli_version
 
-from cli_logging.console_diagnostics import emit_structured_log as _emit_structured_log
-from cli_logging.logging_utils import LOG_LEVEL_MAP, Operation, resolve_log_level
-from parallel_executor import execute_in_parallel
-from throttle_gate import ThrottleGate
-from retry_policy import (
-    is_retryable_status,
-    get_backoff_seconds,
-    get_retry_after_seconds,
+from cli_logging.cli_logger import (
+    CLI_LOGGER,
+    DIAGNOSTIC_RECORDS,
+    configure_cli_logging,
+    emit_structured_log,
 )
+from cli_logging.logging_utils import Operation, resolve_log_level
 
-# Allowed endpoints for URL validation
-ALLOWED_ENDPOINTS = [
-    'substrate.office.com',
-    'graph.microsoft.com',
-]
-
-MAX_CONCURRENCY = 5
-MAX_ATTEMPTS = 4  # Initial attempt + 3 retries
-MAX_TURNS_PER_THREAD = 20
-LONG_THREAD_WARNING_THRESHOLD = 10
-
-
-@dataclass
-class PipelineConfig:
-    """Runtime configuration for the evaluation pipeline."""
-    agent_client: BaseAgentClient
-    model_config: AzureOpenAIModelConfiguration
-    has_azure_openai: bool
-    default_evaluators: Dict[str, Any]
-    chat_gate: ThrottleGate = field(default_factory=lambda: ThrottleGate("chat_api"))
-    is_retryable_status: Any = field(default=is_retryable_status)
-    get_backoff_seconds: Any = field(default=get_backoff_seconds)
-    get_retry_after_seconds: Any = field(default=get_retry_after_seconds)
-
-class CallPath(Enum):
-    """ Enum to indicate which call path to use. """
-    ACCESS_TOKEN = "access_token"
-    COPILOT_AUTH = "copilot_auth"
-    A2A = "a2a"
-
-
-class ItemType(Enum):
-    SINGLE_TURN = "single_turn"
-    MULTI_TURN = "multi_turn"
-
-
-def detect_item_type(item: dict) -> ItemType:
-    """Determine if an evaluation item is single-turn or multi-turn.
-
-    Returns ItemType.SINGLE_TURN if item has 'prompt' without 'turns',
-    ItemType.MULTI_TURN if item has 'turns' array.
-
-    Raises ValueError for invalid items (both, neither, or invalid turns).
-    """
-    has_turns = "turns" in item
-    has_prompt = "prompt" in item
-
-    if has_turns and has_prompt:
-        raise ValueError(
-            "Invalid evaluation item: cannot have both 'turns' and 'prompt'. "
-            "Use 'turns' for multi-turn threads or 'prompt' for single-turn."
-        )
-
-    if has_turns and not isinstance(item["turns"], list):
-        raise ValueError("Invalid evaluation item: 'turns' must be a list")
-
-    if has_turns:
-        if len(item["turns"]) == 0:
-            raise ValueError("Invalid multi-turn thread: 'turns' array cannot be empty")
-        return ItemType.MULTI_TURN
-
-    if has_prompt:
-        return ItemType.SINGLE_TURN
-
-    raise ValueError(
-        "Invalid evaluation item: must have either 'turns' array (multi-turn) "
-        "or 'prompt' field (single-turn)"
-    )
-
-
-# Flags that should bypass remote min-version enforcement.
-# --help is not needed here because argparse exits before runtime checks.
-VERSION_CHECK_BYPASS_FLAGS = (
-    "signout",
+from cli_args import parse_arguments, should_bypass_min_version_check
+from env_validator import (
+    ALLOWED_ENDPOINTS,
+    validate_endpoint_url,
+    validate_environment,
 )
+from common import (
+    ENV_AZURE_AI_OPENAI_ENDPOINT,
+    ENV_AZURE_AI_API_KEY,
+    ENV_AZURE_AI_API_VERSION,
+    ENV_AZURE_AI_MODEL_NAME,
+    ENV_TENANT_ID,
+    ENV_WORK_IQ_A2A_ENDPOINT,
+    ENV_WORK_IQ_A2A_CLIENT_ID,
+    ENV_WORK_IQ_A2A_SCOPES,
+    RunConfig,
+)
+from prompt_loader import get_prompt_datasets
+from agent_selector import select_agent_interactively
+from evaluation_runner import PipelineConfig, run_pipeline
+from result_writer import output_results
 
-CLI_LOGGER_NAME = "m365.eval.cli"
-CLI_LOGGER = logging.getLogger(CLI_LOGGER_NAME)
-DIAGNOSTIC_RECORDS: List[Dict[str, Any]] = []
-
-
-def configure_cli_logging(effective_log_level: str) -> None:
-    if not CLI_LOGGER.handlers:
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setFormatter(logging.Formatter("%(message)s"))
-        CLI_LOGGER.addHandler(handler)
-        CLI_LOGGER.propagate = False
-    CLI_LOGGER.setLevel(LOG_LEVEL_MAP[effective_log_level])
-
-
-def emit_structured_log(level: str, message: str, operation: str = Operation.EVALUATE) -> None:
-    _emit_structured_log(
-        level, message, operation,
-        logger=CLI_LOGGER,
-        diagnostic_records=DIAGNOSTIC_RECORDS,
-    )
-
-
-def should_bypass_min_version_check(args: argparse.Namespace) -> bool:
-    """Return True if the current invocation should skip min-version checks."""
-    return any(getattr(args, flag, False) for flag in VERSION_CHECK_BYPASS_FLAGS)
-
-def write_results_to_html(results: List[Dict], output_file: str,
-                          agent_name: Optional[str] = None, agent_id: Optional[str] = None,
-                          cli_version: Optional[str] = None):
-    """Write results to HTML file using generate_html_report from generate_report.py."""
-    try:
-        html = generate_html_report(results, agent_name=agent_name, agent_id=agent_id,
-                                    cli_version=cli_version)
-        with open(output_file, 'w', encoding='utf-8') as f:
-            f.write(html)
-        emit_structured_log("info", f"HTML report saved to {output_file}", operation=Operation.WRITE_OUTPUT)
-    except Exception as e:
-        emit_structured_log("error", f"Error writing to HTML file: {e}", operation=Operation.WRITE_OUTPUT)
-        sys.exit(1)
-
-def get_default_prompts_and_responses():
-    """Get a list of prompts and responses."""
-    prompts = [
-        "What is Microsoft Graph?"
-    ]
-    expected_responses = [
-        "Microsoft Graph is a gateway to data and intelligence in Microsoft 365."
-    ]
-    return prompts, expected_responses
-
-def load_prompts_from_file(file_path: str) -> Tuple[List[Dict], Optional[Dict]]:
-    """Load prompts and expected responses from a JSON file.
-
-    Supports three formats:
-    1. Eval document: {"schemaVersion": "1.0.0", "items": [{"prompt": "..."}]}
-    2. Array format: [{"prompt": "...", "expected_response": "..."}]
-    3. Dict format: {"prompts": [...], "expected_responses": [...]}
-
-    For eval documents (format 1) and array format (format 2), schema validation
-    and auto-upgrade are applied via DocumentUpgrader.
-
-    Returns:
-        Tuple of (eval_items, default_evaluators). Items are dicts with prompt,
-        expected_response, and optional evaluators/evaluators_mode fields.
-    """
-    try:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            data = json.load(f)
-
-        # Detect if this is an eval document (has "items" key) or could be upgraded
-        is_eval_document = (
-            isinstance(data, dict) and "items" in data
-        ) or isinstance(data, list)
-
-        # Run schema validation and auto-upgrade for eval documents
-        if is_eval_document:
-            try:
-                upgrader = DocumentUpgrader()
-            except Exception as e:
-                # Schema infrastructure not available (missing files, etc.) — skip
-                emit_structured_log("warning", f"Unable to initialize document upgrader: {e}", operation=Operation.LOAD_PROMPTS)
-                upgrader = None
-
-            if upgrader is not None:
-                result = upgrader.upgrade(Path(file_path))
-
-                if result.error:
-                    emit_structured_log("error", f"Schema validation error: {result.error}", operation=Operation.LOAD_PROMPTS)
-                    sys.exit(1)
-
-                if result.upgraded and result.message:
-                    emit_structured_log("info", result.message, operation=Operation.LOAD_PROMPTS)
-
-                # Use the parsed document from the upgrade result
-                if result.document is not None:
-                    data = result.document
-
-        if isinstance(data, list):
-            # Format: [{"prompt": "...", "expected_response": "..."}, ...]
-            return data, None
-        elif isinstance(data, dict):
-            if "items" in data:
-                # Eval document format: {"schemaVersion": "...", "items": [...]}
-                return data["items"], data.get("default_evaluators")
-            else:
-                # Format: {"prompts": [...], "expected_responses": [...]}
-                prompts = data.get("prompts", [])
-                expected_responses = data.get("expected_responses", [])
-                eval_items = [
-                    {"prompt": p, "expected_response": e}
-                    for p, e in zip(prompts, expected_responses)
-                ]
-                return eval_items, None
-        else:
-            raise ValueError("Invalid file format")
-    except SystemExit:
-        raise
-    except Exception as e:
-        emit_structured_log("error", f"Error loading prompts from file: {e}", operation=Operation.LOAD_PROMPTS)
-        sys.exit(1)
-
-def get_interactive_prompts() -> Tuple[List[str], List[str]]:
-    """Get prompts and expected responses interactively."""
-    prompts = []
-    expected_responses = []
-    
-    print("Interactive mode: Enter your prompts and expected responses.")
-    print("Press Enter with empty prompt to finish.")
-    
-    while True:
-        prompt = input(f"\nPrompt {len(prompts) + 1}: ").strip()
-        if not prompt:
-            break
-        
-        expected = input(f"Expected response {len(expected_responses) + 1}: ").strip()
-        
-        prompts.append(prompt)
-        expected_responses.append(expected)
-    
-    if not prompts:
-        print("No prompts entered. Exiting.")
-        sys.exit(1)
-    
-    return prompts, expected_responses
-
-
-_DEFAULT_PASS_THRESHOLD = 3
-
-
-def _decorate_metric(metric_id: str, data, threshold: Optional[int] = None) -> Dict[str, Any]:
-    """Augment raw evaluator output with standardized threshold + pass/fail result."""
-    pass_threshold = threshold if threshold is not None else _DEFAULT_PASS_THRESHOLD
-    payload = {}
-    if isinstance(data, dict):
-        payload.update(data)
-    else:
-        payload['raw'] = data
-
-    score_val = None
-    if isinstance(data, dict):
-        if metric_id in data:
-            score_val = data[metric_id]
-    if isinstance(score_val, (int, float)):
-        payload['threshold'] = pass_threshold
-        payload['result'] = STATUS_PASS if score_val >= pass_threshold else STATUS_FAIL
-    else:
-        payload['threshold'] = pass_threshold
-        payload.setdefault('result', STATUS_UNKNOWN)
-    return payload
-
-
-def _run_evaluators_for_item(
-    prompt: str,
-    actual_response: str,
-    expected_response: str,
-    enhanced_response: Dict[str, Any],
-    resolved_evaluators: Dict[str, Any],
-    model_config: AzureOpenAIModelConfiguration,
-    has_azure_openai: bool,
-    args,
-) -> Tuple[Dict[str, Optional[str]], List[str]]:
-    """Run resolved evaluators against a single item/turn.
-
-    Returns (results_dict, evaluators_ran).
-    """
-    has_tool_defs = bool(
-        args.m365_agent_id and enhanced_response.get("tool_definitions")
-    )
-    available_context = {
-        REQUIRES_AZURE_OPENAI: has_azure_openai,
-        REQUIRES_TOOL_DEFINITIONS: has_tool_defs,
-    }
-
-    results_dict: Dict[str, Optional[str]] = {}
-    evaluators_ran: List[str] = []
-
-    for eval_name, eval_options in resolved_evaluators.items():
-        can_run, warn_msg = check_prerequisites(eval_name, available_context)
-        if not can_run:
-            if warn_msg:
-                emit_structured_log(
-                    "warning",
-                    f"Evaluator '{eval_name}' prerequisite check failed: {warn_msg}",
-                    operation=Operation.EVALUATE,
-                )
-            results_dict[eval_name] = None
-            continue
-
-        threshold = get_evaluator_threshold(eval_name, eval_options)
-
-        try:
-            if eval_name == RELEVANCE:
-                raw_score = RelevanceEvaluator(model_config=model_config)(query=prompt, response=actual_response)
-                results_dict[RELEVANCE] = _decorate_metric(METRIC_IDS[RELEVANCE], raw_score, threshold)
-            elif eval_name == COHERENCE:
-                raw_score = CoherenceEvaluator(model_config=model_config)(query=prompt, response=actual_response)
-                results_dict[COHERENCE] = _decorate_metric(METRIC_IDS[COHERENCE], raw_score, threshold)
-            elif eval_name == GROUNDEDNESS:
-                raw_score = GroundednessEvaluator(model_config=model_config)(response=actual_response, context=expected_response)
-                results_dict[GROUNDEDNESS] = _decorate_metric(METRIC_IDS[GROUNDEDNESS], raw_score, threshold)
-            elif eval_name == TOOL_CALL_ACCURACY:
-                raw_score = ToolCallAccuracyEvaluator(model_config)(
-                    query=prompt,
-                    response=enhanced_response.get("response", actual_response),
-                    tool_definitions=enhanced_response.get("tool_definitions", []),
-                )
-                results_dict[TOOL_CALL_ACCURACY] = _decorate_metric(METRIC_IDS[TOOL_CALL_ACCURACY], raw_score, threshold)
-            elif eval_name == CITATIONS:
-                fmt_str = eval_options.get("citation_format", "oai_unicode")
-                fmt_map = {
-                    "oai_unicode": CitationFormat.OAI_UNICODE,
-                    "bracket": CitationFormat.LEGACY_BRACKET,
-                    "mixed": CitationFormat.AUTO,
-                }
-                raw_score = CitationsEvaluator(citation_format=fmt_map.get(fmt_str, CitationFormat.OAI_UNICODE))(response=actual_response)
-                results_dict[CITATIONS] = _decorate_metric(METRIC_IDS[CITATIONS], raw_score, threshold)
-            elif eval_name == EXACT_MATCH:
-                case_sensitive = eval_options.get("case_sensitive", False)
-                raw_score = ExactMatchEvaluator(case_sensitive=case_sensitive)(response=actual_response, expected_answer=expected_response)
-                # ExactMatch is binary — the evaluator already sets 'result'
-                # so _decorate_metric (which computes result from score vs threshold) is not needed.
-                results_dict[EXACT_MATCH] = raw_score
-            elif eval_name == PARTIAL_MATCH:
-                case_sensitive = eval_options.get("case_sensitive", False)
-                raw_score = PartialMatchEvaluator(case_sensitive=case_sensitive)(response=actual_response, expected_answer=expected_response)
-                results_dict[PARTIAL_MATCH] = _decorate_metric(METRIC_IDS[PARTIAL_MATCH], raw_score, threshold)
-
-            evaluators_ran.append(eval_name)
-        except Exception as e:
-            emit_structured_log(
-                "error",
-                f"Evaluator '{eval_name}' crashed and will be omitted from results: {e}",
-                operation=Operation.EVALUATE,
-            )
-            results_dict[eval_name] = None
-
-    return results_dict, evaluators_ran
-
-
-def _evaluate_single_response(
-    enhanced_response: Dict[str, Any],
-    eval_item: Dict,
-    args,
-    model_config: AzureOpenAIModelConfiguration,
-    has_azure_openai: bool,
-    default_evaluators: Dict[str, Any],
-) -> Dict[str, Any]:
-    """Run all evaluators for a single prompt/response pair and return the result dict."""
-    actual_response_text = get_response_text_for_evaluation(enhanced_response)
-    prompt = eval_item.get("prompt", "")
-    expected_response = eval_item.get("expected_response", "")
-
-    resolved = resolve_evaluators_for_prompt(
-        eval_item.get("evaluators"), eval_item.get("evaluators_mode", "extend"),
-        prompt, default_evaluators,
-    )
-
-    results_dict, evaluators_ran = _run_evaluators_for_item(
-        prompt, actual_response_text, expected_response, enhanced_response,
-        resolved, model_config, has_azure_openai, args,
-    )
-
-    evaluation_result = {
-        "prompt": prompt,
-        "response": enhanced_response.get(
-            "display_response_text", actual_response_text
-        ),
-        "expected_response": expected_response,
-        "evaluators_ran": evaluators_ran,
-        "results": results_dict,
-    }
-
-    if "evaluators" in eval_item:
-        evaluation_result["evaluators"] = eval_item["evaluators"]
-    if "evaluators_mode" in eval_item:
-        evaluation_result["evaluators_mode"] = eval_item["evaluators_mode"]
-
-    if getattr(args, "effective_log_level", "info") == "debug":
-        emit_structured_log(
-            "debug",
-            f"Evaluation completed for prompt='{evaluation_result['prompt']}'. "
-            f"Evaluators: {', '.join(evaluators_ran)}. "
-            f"Scores: {evaluation_result['results']}",
-            operation=Operation.EVALUATE,
-        )
-
-    return evaluation_result
-
-
-def _check_all_passed(results_dict: Dict[str, Optional[Dict[str, Any]]]) -> bool:
-    """Check if all evaluator results passed. Skipped evaluators (None) are ignored."""
-    for result_data in results_dict.values():
-        if result_data is None:
-            continue
-        if result_data.get("result") == STATUS_FAIL:
-            return False
-    return True
-
-def _evaluate_multi_turn_responses(
-    turns: List[Dict],
-    args,
-    default_evaluators: Dict[str, Any],
-    model_config: AzureOpenAIModelConfiguration,
-    has_azure_openai: bool,
-) -> Tuple[List[Dict], Dict]:
-    """Run per-turn evaluations and build evaluated turn results with summary.
-
-    Returns:
-        Tuple of (evaluated_turns, summary). Each evaluated turn contains
-        prompt, response, expected_response, status, evaluators_ran, results,
-        and optionally error. Does not mutate the input turns.
-    """
-    evaluated_turns: List[Dict] = []
-    turns_passed = 0
-    turns_failed = 0
-
-    for i, turn in enumerate(turns):
-        evaluated_turn: Dict[str, Any] = {
-            "prompt": turn.get("prompt", ""),
-        }
-        if "expected_response" in turn:
-            evaluated_turn["expected_response"] = turn["expected_response"]
-        if "response" in turn:
-            evaluated_turn["response"] = turn["response"]
-        if "evaluators" in turn:
-            evaluated_turn["evaluators"] = turn["evaluators"]
-        if "evaluators_mode" in turn:
-            evaluated_turn["evaluators_mode"] = turn["evaluators_mode"]
-
-        if turn.get("status") == STATUS_ERROR:
-            evaluated_turn["status"] = STATUS_ERROR
-            evaluated_turn["error"] = turn.get("error", "")
-            turns_failed += 1
-            evaluated_turns.append(evaluated_turn)
-            continue
-
-        enhanced_response = turn.get("_enhanced_response", {})
-        actual_response = get_response_text_for_evaluation(enhanced_response)
-
-        resolved = resolve_evaluators_for_prompt(
-            turn.get("evaluators"), turn.get("evaluators_mode", "extend"),
-            turn.get("prompt", ""), default_evaluators,
-        )
-
-        results_dict, evaluators_ran = _run_evaluators_for_item(
-            turn.get("prompt", ""), actual_response, turn.get("expected_response", ""),
-            enhanced_response, resolved, model_config, has_azure_openai, args,
-        )
-
-        all_passed = _check_all_passed(results_dict)
-
-        evaluated_turn["results"] = results_dict
-        evaluated_turn["evaluators_ran"] = evaluators_ran
-        evaluated_turn["status"] = STATUS_PASS if all_passed else STATUS_FAIL
-
-        if getattr(args, "effective_log_level", "info") == "debug":
-            emit_structured_log(
-                "debug",
-                f"Evaluation completed for turn {i + 1} prompt='{turn.get('prompt', '')}'. "
-                f"Evaluators: {', '.join(evaluators_ran)}. "
-                f"Scores: {results_dict}",
-                operation=Operation.EVALUATE,
-            )
-
-        if all_passed:
-            turns_passed += 1
-        else:
-            turns_failed += 1
-
-        evaluated_turns.append(evaluated_turn)
-
-    turns_total = len(turns)
-    if turns_passed == turns_total:
-        overall_status = STATUS_PASS
-    elif turns_failed == turns_total:
-        overall_status = STATUS_FAIL
-    else:
-        overall_status = STATUS_PARTIAL
-
-    summary = {
-        "turns_total": turns_total,
-        "turns_passed": turns_passed,
-        "turns_failed": turns_failed,
-        "overall_status": overall_status,
-    }
-
-    return evaluated_turns, summary
-
-
-def get_effective_worker_count(prompt_count: int, args) -> int:
-    """Compute safe worker count for prompt processing."""
-    if prompt_count <= 0:
-        return 1
-
-    requested = getattr(args, "concurrency", 5)
-    try:
-        requested_int = int(requested)
-    except (TypeError, ValueError):
-        requested_int = 5
-
-    bounded = max(1, min(requested_int, MAX_CONCURRENCY))
-    return min(bounded, prompt_count)
-
-
-def run_pipeline(
-    pipeline: PipelineConfig,
-    eval_items: List[Dict],
-    args,
-) -> List[Dict[str, Any]]:
-    """Run the full evaluation pipeline: send prompts and evaluate responses in parallel.
-
-    Each worker processes one prompt end-to-end: send → evaluate.
-    Results are returned in original prompt order (FR-006).
-    """
-    # Validate all evaluator names upfront before dispatching workers
-    all_evaluator_maps = [pipeline.default_evaluators]
-    for eval_item in eval_items:
-        if "evaluators" in eval_item:
-            all_evaluator_maps.append(eval_item["evaluators"])
-        for turn in eval_item.get("turns", []):
-            if "evaluators" in turn:
-                all_evaluator_maps.append(turn["evaluators"])
-    for emap in all_evaluator_maps:
-        validate_evaluator_names(emap)
-
-    # Validate all items upfront and classify types before dispatching workers
-    item_types: List[ItemType] = []
-    for idx, eval_item in enumerate(eval_items):
-        try:
-            item_type = detect_item_type(eval_item)
-        except ValueError as e:
-            raise ValueError(f"Invalid evaluation item at index {idx}: {e}") from e
-        if item_type == ItemType.MULTI_TURN:
-            turn_count = len(eval_item["turns"])
-            if turn_count > MAX_TURNS_PER_THREAD:
-                raise ValueError(
-                    f"Invalid evaluation item at index {idx}: 'turns' array has "
-                    f"{turn_count} items (max {MAX_TURNS_PER_THREAD})"
-                )
-        item_types.append(item_type)
-
-    total = len(eval_items)
-    worker_count = get_effective_worker_count(total, args)
-
-    multi_turn_count = sum(1 for t in item_types if t == ItemType.MULTI_TURN)
-    single_turn_count = total - multi_turn_count
-
-    emit_structured_log(
-        "info",
-        f"Running pipeline with {worker_count} worker(s) for {total} item(s) "
-        f"({single_turn_count} single-turn, {multi_turn_count} multi-turn).",
-        operation=Operation.EVALUATE,
-    )
-
-    def _process_item(eval_item: Dict, index: int) -> Dict[str, Any]:
-        if item_types[index] == ItemType.MULTI_TURN:
-            return _process_multi_turn(eval_item, index)
-        return _process_single_turn(eval_item, index)
-
-    def _process_single_turn(eval_item: Dict, index: int) -> Dict[str, Any]:
-        prompt = eval_item.get("prompt", "")
-        emit_structured_log(
-            "info",
-            f"Processing item {index + 1}/{total} (single-turn).",
-            operation=Operation.SEND_PROMPT,
-        )
-
-        # Phase A: Send prompt to agent (with retry + throttle gate)
-        response = None
-        for attempt in range(1, MAX_ATTEMPTS + 1):
-            pipeline.chat_gate.wait_if_blocked()
-            try:
-                response, _ = pipeline.agent_client.send_prompt(prompt, agent_id=args.m365_agent_id)
-                break
-            except Exception as exc:
-                cause = exc.__cause__
-                status = int(getattr(cause, "code", 0) or 0) or None if cause else None
-                retry_after = get_retry_after_seconds(
-                    cause.headers.get("Retry-After") if cause and getattr(cause, "headers", None) else None
-                )
-
-                if retry_after is not None and pipeline.is_retryable_status(status):
-                    pipeline.chat_gate.apply_retry_after(retry_after)
-
-                if not pipeline.is_retryable_status(status) or attempt >= MAX_ATTEMPTS:
-                    emit_structured_log(
-                        "error",
-                        f"Item {index + 1}/{total} failed after {attempt} attempt(s): {exc}",
-                        operation=Operation.SEND_PROMPT,
-                    )
-                    return {
-                        "prompt": prompt,
-                        "response": "",
-                        "expected_response": eval_item.get("expected_response", ""),
-                        "evaluators_ran": [],
-                        "results": {},
-                        "status": STATUS_ERROR,
-                        "errorDetails": str(exc),
-                    }
-
-                delay = retry_after if retry_after is not None else pipeline.get_backoff_seconds(attempt)
-                time.sleep(delay)
-
-        # Phase B: Evaluate response
-        return _evaluate_single_response(
-            response, eval_item, args,
-            pipeline.model_config, pipeline.has_azure_openai,
-            pipeline.default_evaluators,
-        )
-
-    def _process_multi_turn(eval_item: Dict, index: int) -> Dict[str, Any]:
-        turns = eval_item["turns"]
-        thread_name = eval_item.get("name", "Unnamed thread")
-        emit_structured_log(
-            "info",
-            f"Processing item {index + 1}/{total} (multi-turn: '{thread_name}').",
-            operation=Operation.SEND_PROMPT,
-        )
-
-        if len(turns) > LONG_THREAD_WARNING_THRESHOLD:
-            emit_structured_log(
-                "warning",
-                f"Thread '{thread_name}' has {len(turns)} turns (>{LONG_THREAD_WARNING_THRESHOLD}). This may take a while.",
-                operation=Operation.SEND_PROMPT,
-            )
-
-        # Phase A: Send each turn with throttle gate + 429-only retry
-        # Multi-turn only retries on 429 (server confirmed it didn't process
-        # the request). Other transient errors (503, 504) are ambiguous about
-        # whether the server processed the turn, risking duplicate turns in
-        # the conversation if retried.
-        conversation_context = None
-        conversation_id = None
-        enriched_turns: List[Dict[str, Any]] = []
-        failed = False
-
-        for i, turn in enumerate(turns):
-            prompt = turn["prompt"]
-            emit_structured_log(
-                "debug",
-                f"Sending turn {i + 1}/{len(turns)} of '{thread_name}'.",
-                operation=Operation.SEND_PROMPT,
-            )
-
-            response = None
-            for attempt in range(1, MAX_ATTEMPTS + 1):
-                pipeline.chat_gate.wait_if_blocked()
-                try:
-                    response, conversation_context = pipeline.agent_client.send_prompt(
-                        prompt, agent_id=args.m365_agent_id,
-                        conversation_context=conversation_context,
-                    )
-                    break
-                except Exception as exc:
-                    cause = exc.__cause__
-                    status = int(getattr(cause, "code", 0) or 0) or None if cause else None
-                    retry_after = get_retry_after_seconds(
-                        cause.headers.get("Retry-After") if cause and getattr(cause, "headers", None) else None
-                    )
-
-                    # Only retry on 429 — server confirmed it didn't process the request
-                    if status == 429 and attempt < MAX_ATTEMPTS:
-                        if retry_after is not None:
-                            pipeline.chat_gate.apply_retry_after(retry_after)
-                        delay = retry_after if retry_after is not None else pipeline.get_backoff_seconds(attempt)
-                        time.sleep(delay)
-                        continue
-
-                    # All other errors: stop the thread
-                    emit_structured_log(
-                        "error",
-                        f"Turn {i + 1}/{len(turns)} of '{thread_name}' failed after {attempt} attempt(s): {exc}",
-                        operation=Operation.SEND_PROMPT,
-                    )
-                    failed = True
-                    break
-
-            if failed:
-                # Mark this turn and all remaining turns as error
-                enriched_turns.append({
-                    **turn,
-                    "response": "",
-                    "status": STATUS_ERROR,
-                    "error": "Failed to get response from agent",
-                })
-                for j in range(i + 1, len(turns)):
-                    enriched_turns.append({
-                        **turns[j],
-                        "response": "",
-                        "status": STATUS_ERROR,
-                        "error": "Skipped: preceding turn failed",
-                    })
-                break
-
-            # Enrich turn with response
-            response_text = get_response_text_for_evaluation(response)
-            enriched_turns.append({
-                **turn,
-                "response": response.get("display_response_text", response_text),
-                "_enhanced_response": response,
-            })
-
-            # Capture conversation_id from first response
-            if conversation_id is None:
-                conversation_id = response.get("metadata", {}).get("conversation_id")
-
-        # Phase B: Run per-turn evaluations
-        evaluated_turns, summary = _evaluate_multi_turn_responses(
-            enriched_turns, args, pipeline.default_evaluators,
-            model_config=pipeline.model_config,
-            has_azure_openai=pipeline.has_azure_openai,
-        )
-
-        return {
-            "type": "multi_turn",
-            "name": eval_item.get("name", ""),
-            "description": eval_item.get("description", ""),
-            "conversation_id": conversation_id or "",
-            "turns": evaluated_turns,
-            "summary": summary,
-        }
-
-    execution_results = execute_in_parallel(
-        eval_items, _process_item, max_workers=worker_count,
-    )
-
-    # Unwrap WorkerResult objects into plain dicts, with error fallback
-    ordered_results: List[Dict[str, Any]] = []
-    for wr in execution_results:
-        if wr.error:
-            idx = wr.index
-            item = eval_items[idx]
-            if item_types[idx] == ItemType.MULTI_TURN:
-                ordered_results.append({
-                    "type": "multi_turn",
-                    "name": item.get("name", ""),
-                    "turns": [
-                        {**t, "status": STATUS_ERROR, "error": str(wr.error), "response": "", "results": {}}
-                        for t in item.get("turns", [])
-                    ],
-                    "summary": {
-                        "turns_total": len(item.get("turns", [])),
-                        "turns_passed": 0,
-                        "turns_failed": len(item.get("turns", [])),
-                        "overall_status": STATUS_FAIL,
-                    },
-                    "error": str(wr.error),
-                })
-            else:
-                ordered_results.append({
-                    "prompt": item.get("prompt", ""),
-                    "response": "",
-                    "expected_response": item.get("expected_response", ""),
-                    "evaluators_ran": [],
-                    "results": {},
-                    "status": STATUS_ERROR,
-                    "errorDetails": str(wr.error),
-                })
-        else:
-            ordered_results.append(wr.value)
-
-    return ordered_results
-
-
-
-def write_results_to_console(results, agent_name: Optional[str] = None,
-                             agent_id: Optional[str] = None,
-                             cli_version: Optional[str] = None):
-    """Write the response to console."""
-    # ANSI color codes
-    BOLD = '\033[1m'
-    BLUE = '\033[94m'
-    GREEN = '\033[92m'
-    YELLOW = '\033[93m'
-    CYAN = '\033[96m'
-    MAGENTA = '\033[95m'
-    ORANGE = '\033[38;5;208m'
-    RED = '\033[91m'
-    RESET = '\033[0m'
-
-    def _print_evaluated_item(response: str, expected_response: str,
-                              evaluators_ran: List[str], item_results: Dict[str, Any],
-                              error: Optional[str] = None) -> None:
-        """Print the body of a single evaluated item (single-turn prompt or multi-turn turn).
-
-        The item header (Prompt X / Turn X) is printed by the caller; this helper
-        prints evaluators, response, expected response, error, and metrics.
-        """
-        if evaluators_ran:
-            print(f"{BOLD}{CYAN}Evaluators:{RESET} {', '.join(evaluators_ran)}")
-        if response:
-            print(f"{BOLD}{CYAN}Response:{RESET} {response}")
-        if expected_response:
-            print(f"{BOLD}{YELLOW}Expected Response:{RESET} {expected_response}")
-        if error:
-            print(f"{BOLD}{RED}Error:{RESET} {error}")
-
-        for eval_name, v in item_results.items():
-            if v is None:
-                continue
-            display_name = pascal_case_to_title(eval_name)
-            if eval_name == RELEVANCE:
-                color = MAGENTA
-            elif eval_name == COHERENCE:
-                color = ORANGE
-            else:
-                color = BLUE
-            print(f"{BOLD}{color}{display_name}:{RESET} {json.dumps(v, indent=4)}")
-
-    # Show metadata
-    metadata_parts = []
-    if agent_name:
-        metadata_parts.append(f"Agent Name: {agent_name}")
-    if agent_id:
-        metadata_parts.append(f"Agent ID: {agent_id}")
-    if cli_version:
-        metadata_parts.append(f"CLI Version: {cli_version}")
-    if metadata_parts:
-        print(f"{BOLD}{CYAN}{' | '.join(metadata_parts)}{RESET}")
-        print()
-
-    aggregates = calculate_aggregate_statistics(results)
-    if aggregates:
-        total_items = aggregates[next(iter(aggregates))].get('total_prompts', len(results))
-        if total_items > 1:
-            print(f"{BOLD}{BLUE}Aggregate Statistics ({total_items} prompts):{RESET}")
-            print(f"{BLUE}{'=' * 60}{RESET}")
-
-            for metric_name, stats in aggregates.items():
-                pass_color = GREEN if stats['pass_rate'] >= 80 else YELLOW if stats['pass_rate'] >= 60 else RED
-                prompts_evaluated = stats.get('prompts_evaluated', stats['total_evaluated'])
-                total_prompts = stats.get('total_prompts', total_items)
-                print(f"{BOLD}{CYAN}{metric_name}:{RESET} ({prompts_evaluated}/{total_prompts} prompts)")
-                print(f"  Pass Rate: {pass_color}{stats['pass_rate']:.1f}%{RESET} ({stats['pass_count']}/{stats['total_evaluated']} passed)")
-                print(f"  Avg Score: {MAGENTA}{stats['avg_score']:.2f}{RESET}")
-                if stats.get('threshold') is not None:
-                    print(f"  Threshold: {YELLOW}{stats['threshold']}{RESET}")
-                print()
-
-            print(f"{BLUE}{'=' * 60}{RESET}")
-            print()
-
-    print(f"{BOLD}{BLUE}Individual Results:{RESET}")
-    print(f"{BLUE}{'=' * 50}{RESET}")
-    for i, result in enumerate(results, 1):
-        if result.get("type") == "multi_turn":
-            thread_name = result.get("name", "Unnamed Thread")
-            summary = result.get("summary", {})
-            status = summary.get("overall_status", STATUS_UNKNOWN)
-            status_color = GREEN if status == STATUS_PASS else YELLOW if status == STATUS_PARTIAL else RED
-
-            print(f"{BOLD}{MAGENTA}Thread {i}: {thread_name}{RESET}")
-            for t_idx, turn in enumerate(result.get("turns", []), 1):
-                turn_status = turn.get("status", STATUS_UNKNOWN)
-                turn_color = GREEN if turn_status == STATUS_PASS else RED if turn_status in (STATUS_FAIL, STATUS_ERROR) else YELLOW
-                print(f"{BOLD}{turn_color}Turn {t_idx}:{RESET} [{turn_status}] {turn.get('prompt', '')}")
-                _print_evaluated_item(
-                    response=turn.get("response", ""),
-                    expected_response=turn.get("expected_response", ""),
-                    evaluators_ran=turn.get("evaluators_ran", []),
-                    item_results=turn.get("results", {}),
-                    error=turn.get("error"),
-                )
-            print()
-            print(f"{BOLD}{MAGENTA}Thread {i} Summary:{RESET}")
-            print(f"  Status: {status_color}{status.upper()}{RESET}")
-            print(f"  Turns passed: {status_color}{summary.get('turns_passed', 0)}/{summary.get('turns_total', 0)}{RESET}")
-            print(f"{BLUE}{'-' * 30}{RESET}")
-        else:
-            print(f"{BOLD}{GREEN}Prompt {i}:{RESET} {result['prompt']}")
-            _print_evaluated_item(
-                response=result.get('response', ''),
-                expected_response=result.get('expected_response', ''),
-                evaluators_ran=result.get('evaluators_ran', []),
-                item_results=result.get('results', {}),
-                error=result.get('errorDetails'),
-            )
-            print(f"{BLUE}{'-' * 30}{RESET}")
-
-def extract_eval_score(data: dict, metric_id: str) -> Optional[Dict]:
-    """Extract an EvalScore object from a decorated metric dict.
-
-    Maps internal decorated-metric format to schema EvalScore:
-    {score, result, threshold} (required) + reason, evaluator (optional).
-    """
-    DEFAULT_THRESHOLD = 3  # fallback; decorate_metric should always set this
-
-    score_val = None
-    if metric_id in data and isinstance(data[metric_id], (int, float)):
-        score_val = data[metric_id]
-    if score_val is None:
-        return None
-
-    result = data.get("result")
-    if result not in (STATUS_PASS, STATUS_FAIL):
-        result = STATUS_PASS if score_val >= data.get("threshold", DEFAULT_THRESHOLD) else STATUS_FAIL
-
-    eval_score: Dict[str, Any] = {
-        "score": score_val,
-        "result": result,
-        "threshold": data.get("threshold", DEFAULT_THRESHOLD),
-    }
-    reason = data.get(f"{metric_id}_reason") or data.get("reason")
-    if reason:
-        eval_score["reason"] = reason
-    return eval_score
-
-
-def _convert_scores_to_schema(results_dict: Dict[str, Any]) -> Dict[str, Any]:
-    """Convert raw evaluator results to schema-compliant score objects.
-
-    Evaluator results in results_dict are dicts (from _decorate_metric) or
-    None when skipped/crashed. None values are omitted from output.
-    """
-    scores: Dict[str, Any] = {}
-
-    for eval_key, schema_key in [
-        (RELEVANCE, "relevance"),
-        (COHERENCE, "coherence"),
-        (GROUNDEDNESS, "groundedness"),
-        (TOOL_CALL_ACCURACY, "toolCallAccuracy"),
-    ]:
-        data = results_dict.get(eval_key)
-        if data is None:
-            continue
-        eval_score = extract_eval_score(data, METRIC_IDS[eval_key])
-        if eval_score:
-            scores[schema_key] = eval_score
-
-    data = results_dict.get(CITATIONS)
-    if data is not None:
-        count = data.get("citations", 0)
-        cit_result = data.get("result")
-        if cit_result not in (STATUS_PASS, STATUS_FAIL):
-            cit_result = STATUS_PASS if count >= data.get("threshold", 1) else STATUS_FAIL
-        citation_score: Dict[str, Any] = {
-            "count": count,
-            "result": cit_result,
-            "threshold": data.get("threshold", 1),
-        }
-        if "citation_format" in data:
-            citation_score["format"] = data["citation_format"]
-        scores["citations"] = citation_score
-
-    data = results_dict.get(EXACT_MATCH)
-    if data is not None:
-        is_match = data.get("exact_match", 0.0) == 1.0
-        scores["exactMatch"] = {
-            "match": is_match,
-            "result": data.get("result", STATUS_PASS if is_match else STATUS_FAIL),
-            "reason": data.get("exact_match_reason", ""),
-        }
-
-    data = results_dict.get(PARTIAL_MATCH)
-    if data is not None:
-        scores["partialMatch"] = {
-            "score": data.get("partial_match", 0.0),
-            "result": data.get("result", STATUS_FAIL),
-            "threshold": data.get("threshold", 0.5),
-            "reason": data.get("partial_match_reason", ""),
-        }
-
-    return scores
-
-
-def convert_result_to_eval_item(result: Dict) -> Dict:
-    """Convert an internal evaluation result dict to a schema-compliant EvalItem."""
-    item: Dict[str, Any] = {
-        "prompt": result["prompt"],
-        "response": result["response"],
-        "expected_response": result["expected_response"],
-    }
-
-    if "evaluators" in result:
-        item["evaluators"] = result["evaluators"]
-    if "evaluators_mode" in result:
-        item["evaluators_mode"] = result["evaluators_mode"]
-
-    scores = _convert_scores_to_schema(result.get("results", {}))
-    if scores:
-        item["scores"] = scores
-
-    return item
-
-
-def convert_thread_result_to_output(thread_result: Dict) -> Dict:
-    """Convert a multi-turn thread result to the output format."""
-    output_turns = []
-    for turn in thread_result.get("turns", []):
-        output_turn: Dict[str, Any] = {"prompt": turn.get("prompt", "")}
-        if "expected_response" in turn:
-            output_turn["expected_response"] = turn["expected_response"]
-        if "response" in turn:
-            output_turn["response"] = turn["response"]
-        if "status" in turn:
-            output_turn["status"] = turn["status"]
-        if "error" in turn:
-            output_turn["error"] = turn["error"]
-        if "evaluators" in turn:
-            output_turn["evaluators"] = turn["evaluators"]
-        if "evaluators_mode" in turn:
-            output_turn["evaluators_mode"] = turn["evaluators_mode"]
-
-        scores = _convert_scores_to_schema(turn.get("results", {}))
-        if scores:
-            output_turn["scores"] = scores
-
-        output_turns.append(output_turn)
-
-    output: Dict[str, Any] = {}
-    if thread_result.get("name"):
-        output["name"] = thread_result["name"]
-    if thread_result.get("description"):
-        output["description"] = thread_result["description"]
-    if thread_result.get("conversation_id"):
-        output["conversation_id"] = thread_result["conversation_id"]
-    output["turns"] = output_turns
-    if thread_result.get("summary"):
-        output["summary"] = thread_result["summary"]
-
-    return output
-
-
-def convert_result_to_output_item(result: Dict) -> Dict:
-    """Convert an internal result dict to an output item. Routes by type."""
-    if result.get("type") == "multi_turn":
-        return convert_thread_result_to_output(result)
-    return convert_result_to_eval_item(result)
-
-
-def write_results_to_json(results: List[Dict], output_file: str, agent_id: Optional[str] = None,
-                          default_evaluators: Optional[Dict[str, Any]] = None,
-                          agent_name: Optional[str] = None,
-                          cli_version: Optional[str] = None):
-    """Write results to a schema-compliant eval document JSON file.
-
-    Output follows the eval-document.schema.json format:
-    {schemaVersion, metadata, default_evaluators?, items: [EvalItem]}
-    """
-    try:
-        try:
-            current_version = SchemaVersionManager().get_current_version()
-        except Exception:
-            current_version = "1.0.0"
-
-        items = [convert_result_to_output_item(r) for r in results]
-
-        metadata: Dict[str, Any] = {
-            "evaluatedAt": datetime.now(timezone.utc).isoformat(),
-        }
-        if agent_id:
-            metadata["agentId"] = agent_id
-        if agent_name:
-            metadata["agentName"] = agent_name
-        if cli_version:
-            metadata["cliVersion"] = cli_version
-
-        output_data: Dict[str, Any] = {
-            "schemaVersion": current_version,
-            "metadata": metadata,
-        }
-
-        if default_evaluators is not None:
-            output_data["default_evaluators"] = default_evaluators
-
-        output_data["items"] = items
-
-        with open(output_file, 'w', encoding='utf-8') as f:
-            json.dump(output_data, f, indent=2, ensure_ascii=False)
-        emit_structured_log("info", f"Results saved to {output_file}", operation=Operation.WRITE_OUTPUT)
-    except Exception as e:
-        emit_structured_log("error", f"Error writing to JSON file: {e}", operation=Operation.WRITE_OUTPUT)
-        sys.exit(1)
-
-def _results_to_csv_json(results_dict: Dict) -> str:
-    """Serialize evaluator results dict to a CSV-safe JSON string.
-
-    Skips None (crashed/skipped evaluators). Results are dicts produced
-    by _decorate_metric.
-    """
-    if not results_dict:
-        return ""
-    non_null = {k: v for k, v in results_dict.items() if v is not None}
-    return json.dumps(non_null) if non_null else ""
-
-
-def write_results_to_csv(results: List[Dict], output_file: str,
-                         agent_name: Optional[str] = None, agent_id: Optional[str] = None,
-                         cli_version: Optional[str] = None):
-    """Write results to CSV file."""
-    try:
-        with open(output_file, 'w', newline='', encoding='utf-8') as f:
-            if results:
-                metadata_parts = []
-                if agent_name:
-                    metadata_parts.append(f"Agent Name: {agent_name}")
-                if agent_id:
-                    metadata_parts.append(f"Agent ID: {agent_id}")
-                if cli_version:
-                    metadata_parts.append(f"CLI Version: {cli_version}")
-                if metadata_parts:
-                    f.write(f"# {' | '.join(metadata_parts)}\n")
-
-                aggregates = calculate_aggregate_statistics(results)
-                if aggregates:
-                    total_items = aggregates[next(iter(aggregates))].get('total_prompts', len(results))
-                    if total_items > 1:
-                        f.write("# AGGREGATE STATISTICS\n")
-                        f.write("Metric,Prompts Evaluated,Total Prompts,Pass Rate (%),Passed,Failed,Avg Score,Threshold\n")
-                        for metric_name, stats in aggregates.items():
-                            threshold_str = str(stats.get('threshold', 'N/A'))
-                            prompts_evaluated = stats.get('prompts_evaluated', stats['total_evaluated'])
-                            total_prompts = stats.get('total_prompts', total_items)
-                            f.write(f"{metric_name},{prompts_evaluated},{total_prompts},{stats['pass_rate']:.1f},{stats['pass_count']},{stats['fail_count']},{stats['avg_score']:.2f},{threshold_str}\n")
-                        f.write("\n# INDIVIDUAL RESULTS\n")
-
-                single_turn_rows = []
-                multi_turn_rows = []
-                for result in results:
-                    if result.get("type") == "multi_turn":
-                        thread_name = result.get("name", "")
-                        for turn_idx, turn in enumerate(result.get("turns", [])):
-                            multi_turn_rows.append({
-                                "thread_name": thread_name,
-                                "turn_index": turn_idx + 1,
-                                "prompt": turn.get("prompt", ""),
-                                "response": turn.get("response", ""),
-                                "expected_response": turn.get("expected_response", ""),
-                                "status": turn.get("status", ""),
-                                "error": turn.get("error", ""),
-                                "scores": _results_to_csv_json(turn.get("results", {})),
-                            })
-                        summary = result.get("summary", {})
-                        multi_turn_rows.append({
-                            "thread_name": thread_name,
-                            "turn_index": "summary",
-                            "prompt": "",
-                            "response": "",
-                            "expected_response": "",
-                            "status": summary.get("overall_status", ""),
-                            "scores": f"{summary.get('turns_passed', 0)}/{summary.get('turns_total', 0)} turns passed",
-                        })
-                    else:
-                        exclude_keys = {'evaluators_ran', 'evaluators', 'evaluators_mode', '_enhanced_response', 'results'}
-                        row = {k: v for k, v in result.items() if k not in exclude_keys}
-                        if "results" in result:
-                            row["scores"] = _results_to_csv_json(result["results"])
-                        single_turn_rows.append(row)
-
-                if single_turn_rows:
-                    if multi_turn_rows:
-                        f.write("# SINGLE-TURN RESULTS\n")
-                    fieldnames = list(single_turn_rows[0].keys())
-                    for row in single_turn_rows:
-                        for k in row:
-                            if k not in fieldnames:
-                                fieldnames.append(k)
-                    writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
-                    writer.writeheader()
-                    writer.writerows(single_turn_rows)
-
-                if multi_turn_rows:
-                    if single_turn_rows:
-                        f.write("\n")
-                    f.write("# MULTI-TURN RESULTS\n")
-                    fieldnames = ["thread_name", "turn_index", "prompt", "response", "expected_response", "status", "error", "scores"]
-                    writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
-                    writer.writeheader()
-                    writer.writerows(multi_turn_rows)
-            emit_structured_log("info", f"Results saved to {output_file}", operation=Operation.WRITE_OUTPUT)
-    except Exception as e:
-        emit_structured_log("error", f"Error writing to CSV file: {e}", operation=Operation.WRITE_OUTPUT)
-        sys.exit(1)
-
-def normalize_agent_id(agent_id):
-    """Append '.declarativeAgent' if agent_id has no '.', else return unchanged.
-
-    Returns the input unchanged when it is None/empty or already contains a dot.
-    """
-    if not agent_id:
-        return agent_id
-    return agent_id if '.' in agent_id else f"{agent_id}.declarativeAgent"
-
-
-def parse_arguments():
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="M365 Copilot Agent Evaluation CLI",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Run with default prompts
-  python main.py
-
-  # Run with custom prompts
-  python main.py --prompts "What is Microsoft Graph?" --expected "Microsoft Graph is a gateway..."
-
-  # Run with prompts from file
-  python main.py --prompts-file prompts.json
-
-  # Interactive mode
-  python main.py --interactive
-
-  # Save results to JSON
-  python main.py --output results.json
-
-  # Save results to CSV
-  python main.py --output results.csv
-
-  # Save results to HTML and open in browser
-  python main.py --output report.html
-
-  # Debug-level diagnostics
-  python main.py --log-level debug
-
-  # Sign out and clear cached authentication tokens
-  python main.py --signout
-        """
-    )
-    
-    # Input options (mutually exclusive)
-    input_group = parser.add_mutually_exclusive_group()
-    input_group.add_argument(
-        '--prompts', 
-        nargs='+', 
-        help='List of prompts to evaluate'
-    )
-    input_group.add_argument(
-        '--prompts-file', 
-        type=str, 
-        help='JSON file containing prompts and expected responses'
-    )
-    input_group.add_argument(
-        '--interactive', 
-        action='store_true', 
-        help='Interactive mode to enter prompts'
-    )
-    
-    # Expected responses (only used with --prompts)
-    parser.add_argument(
-        '--expected', 
-        nargs='+', 
-        help='List of expected responses (must match number of prompts)'
-    )
-    
-    # Agent ID (--m365-agent-id is primary, --agent-id kept for backward compatibility)
-    parser.add_argument(
-        '--m365-agent-id', '--agent-id',
-        type=str,
-        default=os.environ.get("M365_AGENT_ID") or os.environ.get("AGENT_ID"),
-        help='Agent ID (default from M365_AGENT_ID environment variable)'
-    )
-
-    # Output options
-    parser.add_argument(
-        '--output', 
-        type=str, 
-        help='Output file path. Format is determined by file extension: .json, .csv, .html. If not provided, results are printed to console.'
-    )
-    
-    # Behavior options
-    parser.add_argument(
-        '--log-level',
-        nargs='?',
-        const='info',
-        action='append',
-        help='Set log verbosity: debug, info, warning, error. Bare --log-level resolves to info.'
-    )
-
-    parser.add_argument(
-        '--signout',
-        action='store_true',
-        help='Sign out and clear cached authentication tokens'
-    )
-
-    parser.add_argument(
-        '--concurrency',
-        type=int,
-        default=5,
-        help='Number of parallel workers for prompt processing (1-5, default: 5)'
-    )
-    
-    args = parser.parse_args()
-
-    args.m365_agent_id = normalize_agent_id(args.m365_agent_id)
-
-    if args.concurrency < 1:
-        parser.error('--concurrency must be an integer >= 1.')
-    if args.concurrency > MAX_CONCURRENCY:
-        emit_structured_log(
-            "warning",
-            f"--concurrency {args.concurrency} exceeds max {MAX_CONCURRENCY}; clamping to {MAX_CONCURRENCY}.",
-            operation=Operation.SETUP,
-        )
-        args.concurrency = MAX_CONCURRENCY
-
-    return args
-
-def validate_environment() -> CallPath:
-    """Validate required environment variables."""
-    required_env_vars = [
-        "AZURE_AI_OPENAI_ENDPOINT",
-        "AZURE_AI_API_KEY",
-        "AZURE_AI_API_VERSION",
-        "AZURE_AI_MODEL_NAME",
-    ]
-
-    if os.environ.get("COPILOT_API_ACCESS_TOKEN"):
-        call_path = CallPath.ACCESS_TOKEN
-        required_env_vars.extend([
-            "COPILOT_API_ACCESS_TOKEN",
-            "COPILOT_API_ENDPOINT",
-            "X_SCENARIO_HEADER",
-        ])
-    elif os.environ.get("WORK_IQ_A2A_ENDPOINT"):
-        call_path = CallPath.A2A
-        required_env_vars.extend([
-            "WORK_IQ_A2A_ENDPOINT",
-            "WORK_IQ_A2A_CLIENT_ID",
-            "TENANT_ID",
-        ])
-    else:
-        call_path = CallPath.COPILOT_AUTH
-        required_env_vars.extend([
-            "COPILOT_API_ENDPOINT",
-            "X_SCENARIO_HEADER",
-            "M365_EVAL_CLIENT_ID",
-            "TENANT_ID",
-        ])
-    
-    missing_vars = [var for var in required_env_vars if not os.environ.get(var)]
-    if missing_vars:
-        emit_structured_log(
-            "error",
-            "Missing required environment variables: "
-            f"{', '.join(missing_vars)}. Please ensure your .env file contains "
-            "all required Azure configuration.",
-            operation=Operation.VALIDATE_ENV,
-        )
-        sys.exit(1)
-    return call_path
-
-def validate_endpoint_url(url: str, allowed_domains: List[str]) -> bool:
-    """Validate URL against security requirements."""
-    try:
-        parsed = urllib.parse.urlparse(url)
-        
-        # Check for dangerous schemes
-        if parsed.scheme in ['javascript', 'data']:
-            raise ValueError(f"Dangerous URL scheme detected: {parsed.scheme}")
-        
-        # Check for HTTPS requirement
-        if parsed.scheme != 'https':
-            raise ValueError(f"Only HTTPS URLs are allowed, got: {parsed.scheme}")
-        
-        # Check if domain is in allowed list
-        if parsed.netloc not in allowed_domains:
-            raise ValueError(f"Domain not in allowed list: {parsed.netloc}")
-        
-        # Reject fragment URLs
-        if parsed.fragment:
-            raise ValueError("Fragment URLs are not allowed")
-        
-        return True
-        
-    except ValueError:
-        # Re-raise ValueError exceptions
-        raise
-    except Exception as e:
-        # Convert other parsing errors to ValueError
-        raise ValueError(f"Invalid URL format: {url}") from e
-
-def get_prompt_datasets(args) -> Tuple[List[Dict], Optional[Dict]]:
-    """Get prompts and expected responses based on command line arguments.
-
-    Returns:
-        Tuple of (eval_items, default_evaluators).
-    """
-    if args.prompts:
-        if args.expected and len(args.prompts) != len(args.expected):
-            emit_structured_log(
-                "error",
-                "Number of prompts must match number of expected responses. "
-                "Update --expected values to match the prompt count.",
-            )
-            sys.exit(1)
-        expected_responses = args.expected or [""] * len(args.prompts)
-        eval_items = [
-            {"prompt": p, "expected_response": e}
-            for p, e in zip(args.prompts, expected_responses)
-        ]
-        return eval_items, None
-    elif args.prompts_file:
-        return load_prompts_from_file(args.prompts_file)
-    elif args.interactive:
-        prompts, expected_responses = get_interactive_prompts()
-        eval_items = [
-            {"prompt": p, "expected_response": e}
-            for p, e in zip(prompts, expected_responses)
-        ]
-        return eval_items, None
-    else:
-        prompts, expected_responses = get_default_prompts_and_responses()
-        eval_items = [
-            {"prompt": p, "expected_response": e}
-            for p, e in zip(prompts, expected_responses)
-        ]
-        return eval_items, None
-
-def select_agent_interactively(agents: List[Dict[str, Any]]) -> Tuple[Optional[str], Optional[str]]:
-    """
-    Display an interactive agent selector using questionary.
-
-    Args:
-        agents: List of agent dictionaries.
-
-    Returns:
-        Tuple of (agent_id, agent_name) or (None, None) if cancelled/skipped
-    """
-    if not agents:
-        return None, None
-
-    # Build id→name lookup and choices
-    id_to_name: Dict[str, str] = {}
-    choices = []
-    sorted_agents = sorted(agents, key=lambda x: (not x.get('isOwner', False), x.get('name', '')))
-    for agent in sorted_agents:
-        agent_id = agent.get("gptId", "Unknown")
-        agent_name = agent.get("name", "Unknown")
-        agent_description = agent.get("description", "Unknown")
-        agent_is_owner = agent.get('isOwner')
-        id_to_name[agent_id] = agent_name
-
-        # Format the display text
-        display_text = f"{agent_name} ({agent_id}, IsOwner: {agent_is_owner}) - {agent_description}"
-
-        choices.append(questionary.Choice(title=display_text, value=agent_id))
-
-    # Display the selection prompt
-    selected_agent = questionary.select(
-        "Select an agent to evaluate:",
-        choices=choices,
-        use_shortcuts=True,
-        use_arrow_keys=True
-    ).ask()
-
-    return selected_agent, id_to_name.get(selected_agent) if selected_agent else None
+from dataclasses import replace
 
 
-def output_results(results: List[Dict], args, default_evaluators: Optional[Dict[str, Any]] = None,
-                    agent_name: Optional[str] = None, cli_version: Optional[str] = None):
-    """Output results based on specified format."""
-    metadata_kwargs = dict(
-        agent_name=agent_name,
-        agent_id=getattr(args, 'm365_agent_id', None),
-        cli_version=cli_version,
-    )
-    if args.output:
-        output_lower = args.output.lower()
-        if output_lower.endswith('.json'):
-            write_results_to_json(results, args.output, default_evaluators=default_evaluators,
-                                  **metadata_kwargs)
-        elif output_lower.endswith('.csv'):
-            write_results_to_csv(results, args.output, **metadata_kwargs)
-        elif output_lower.endswith('.html'):
-            write_results_to_html(results, args.output, **metadata_kwargs)
-            abs_path = os.path.abspath(args.output)
-            webbrowser.open(f'file://{abs_path}')
-        else:
-            write_results_to_json(results, args.output, default_evaluators=default_evaluators,
-                                  **metadata_kwargs)
-    else:
-        write_results_to_console(results, **metadata_kwargs)
-
 def main():
     """Main function to orchestrate the evaluation process."""
     load_dotenv()
@@ -1555,136 +67,78 @@ def main():
         )
         sys.exit(2)
 
-    args.effective_log_level = effective_log_level
-    configure_cli_logging(effective_log_level)
-    emit_structured_log("info", f"Log level set to '{effective_log_level}'.", operation=Operation.SETUP)
+    config = replace(
+        RunConfig.from_namespace(args),
+        effective_log_level=effective_log_level,
+    )
+    configure_cli_logging(config.effective_log_level)
+    emit_structured_log("info", f"Log level set to '{config.effective_log_level}'.", operation=Operation.SETUP)
 
     # Check minimum version before proceeding
-    quiet_for_version = effective_log_level in ("warning", "error")
+    quiet_for_version = config.effective_log_level in ("warning", "error")
     cli_version = get_cli_version(quiet=quiet_for_version)
-    if not should_bypass_min_version_check(args) and not check_min_version(cli_version, quiet=quiet_for_version):
+    if not should_bypass_min_version_check(config) and not check_min_version(cli_version, quiet=quiet_for_version):
         sys.exit(1)
 
     # Validate environment variables required for evaluation
-    call_path = validate_environment()
-    
-    user_oid = ""
+    validate_environment()
 
-    match call_path:
-        case CallPath.ACCESS_TOKEN:
-            access_token = os.environ["COPILOT_API_ACCESS_TOKEN"]
-            user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
-            copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
-            validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
-            agent_client = SydneyClient(
-                copilot_api_endpoint=copilot_api_endpoint,
-                access_token=access_token,
-                user_oid=user_oid,
-                logger=CLI_LOGGER,
-                diagnostic_records=DIAGNOSTIC_RECORDS,
-            )
+    a2a_endpoint = os.environ[ENV_WORK_IQ_A2A_ENDPOINT]
+    validate_endpoint_url(a2a_endpoint, ALLOWED_ENDPOINTS)
 
-        case CallPath.A2A:
+    a2a_scopes_str = os.environ.get(ENV_WORK_IQ_A2A_SCOPES, "")
+    a2a_auth_handler = AuthHandler(
+        client_id=os.environ[ENV_WORK_IQ_A2A_CLIENT_ID],
+        tenant_id=os.environ[ENV_TENANT_ID],
+        scopes_str=a2a_scopes_str,
+    )
+    if config.signout:
+        try:
+            a2a_auth_handler.clear_cache()
+        except Exception as e:
             emit_structured_log(
-                "warning",
-                "The A2A endpoint is experimental and may change without notice.",
-                operation=Operation.SETUP,
-            )
-            a2a_endpoint = os.environ["WORK_IQ_A2A_ENDPOINT"]
-            validate_endpoint_url(a2a_endpoint, ALLOWED_ENDPOINTS)
-
-            a2a_scopes_str = os.environ.get("WORK_IQ_A2A_SCOPES", "")
-            a2a_auth_handler = AuthHandler(
-                client_id=os.environ["WORK_IQ_A2A_CLIENT_ID"],
-                tenant_id=os.environ["TENANT_ID"],
-                scopes_str=a2a_scopes_str,
-            )
-            try:
-                a2a_auth_result = a2a_auth_handler.acquire_token_interactive() or {}
-                a2a_access_token = a2a_auth_result.get("access_token") or ""
-                if not a2a_access_token:
-                    raise RuntimeError("Failed to acquire A2A access token")
-            except Exception as e:
-                emit_structured_log(
-                    "error",
-                    f"Error during A2A authentication: {e}",
-                    operation=Operation.AUTHENTICATE,
-                )
-                if effective_log_level == "debug":
-                    import traceback
-                    traceback.print_exc()
-                sys.exit(1)
-            try:
-                agent_client = A2AClient(
-                    a2a_endpoint=a2a_endpoint,
-                    access_token=a2a_access_token,
-                    logger=CLI_LOGGER,
-                    diagnostic_records=DIAGNOSTIC_RECORDS,
-                )
-            except Exception as e:
-                emit_structured_log(
-                    "error",
-                    f"Failed to initialize A2A client: {e}",
-                    operation=Operation.SETUP,
-                )
-                sys.exit(1)
-
-        case CallPath.COPILOT_AUTH:
-            copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
-            validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
-            auth_handler = AuthHandler(
-                client_id=os.environ["M365_EVAL_CLIENT_ID"],
-                tenant_id=os.environ["TENANT_ID"],
-                scopes_str=os.environ.get("COPILOT_SCOPES", ""),
+                "error",
+                f"Error during signout: {e}",
+                operation=Operation.AUTHENTICATE,
             )
+            sys.exit(1)
+        sys.exit(0)
 
-            # Signout user
-            if args.signout:
-                try:
-                    auth_handler.clear_cache()
-                except Exception as e:
-                    emit_structured_log("error", f"Error during signout: {e}", operation=Operation.AUTHENTICATE)
-                    sys.exit(1)
-                sys.exit(0)
-
-            try:
-                auth_result = auth_handler.acquire_token_interactive() or {}
-                access_token = auth_result.get("access_token") or ""
-                if not access_token:
-                    raise RuntimeError("Failed to acquire access token from authentication result")
-
-                id_token_claims = auth_result.get("id_token_claims")
-                if not isinstance(id_token_claims, dict):
-                    emit_structured_log(
-                        "warning", "id_token_claims is missing or invalid in authentication result",
-                        operation=Operation.AUTHENTICATE,
-                    )
-                else:
-                    user_oid = id_token_claims.get("oid") or ""
-
-                if not user_oid:
-                    user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
-
-            except Exception as e:
-                emit_structured_log("error", f"Error during authentication: {e}", operation=Operation.AUTHENTICATE)
-                if effective_log_level == "debug":
-                    import traceback
-                    traceback.print_exc()
-                sys.exit(1)
-
-            agent_client = SydneyClient(
-                copilot_api_endpoint=copilot_api_endpoint,
-                access_token=access_token,
-                user_oid=user_oid,
-                logger=CLI_LOGGER,
-                diagnostic_records=DIAGNOSTIC_RECORDS,
-            )
+    try:
+        a2a_auth_result = a2a_auth_handler.acquire_token_interactive() or {}
+        a2a_access_token = a2a_auth_result.get("access_token") or ""
+        if not a2a_access_token:
+            raise RuntimeError("Failed to acquire A2A access token")
+    except Exception as e:
+        emit_structured_log(
+            "error",
+            f"Error during A2A authentication: {e}",
+            operation=Operation.AUTHENTICATE,
+        )
+        if config.effective_log_level == "debug":
+            traceback.print_exc()
+        sys.exit(1)
+    try:
+        agent_client = A2AClient(
+            a2a_endpoint=a2a_endpoint,
+            access_token=a2a_access_token,
+            token_refresh_fn=make_token_refresh_fn(a2a_auth_handler),
+            logger=CLI_LOGGER,
+            diagnostic_records=DIAGNOSTIC_RECORDS,
+        )
+    except Exception as e:
+        emit_structured_log(
+            "error",
+            f"Failed to initialize A2A client: {e}",
+            operation=Operation.SETUP,
+        )
+        sys.exit(1)
 
     # 1. Load evaluation datasets
-    eval_items, file_default_evaluators = get_prompt_datasets(args)
+    eval_items, file_default_evaluators = get_prompt_datasets(config)
     default_evaluators = resolve_default_evaluators(file_default_evaluators)
 
-    if effective_log_level in ("info", "debug"):
+    if config.effective_log_level in ("info", "debug"):
         multi_turn_count = sum(1 for item in eval_items if "turns" in item)
         single_turn_count = len(eval_items) - multi_turn_count
         emit_structured_log(
@@ -1697,54 +151,53 @@ def main():
     agent_name = None
     try:
         # 2. Agent selection - when no agent ID is provided, discover agents
-        # via the active client (A2A or REST) and prompt interactively.
-        if not args.m365_agent_id:
-            if effective_log_level in ("info", "debug"):
+        # via the active client (A2A) and prompt interactively.
+        if not config.m365_agent_id:
+            if config.effective_log_level in ("info", "debug"):
                 emit_structured_log("info", "No agent ID provided. Fetching available agents.", operation=Operation.FETCH_AGENTS)
 
             available_agents = agent_client.fetch_available_agents()
             if not available_agents:
                 emit_structured_log(
-                  "error",
-                  "No agents are available for interactive selection. Re-run with "
-                  "--m365-agent-id or set M365_AGENT_ID.",
-                  operation=Operation.FETCH_AGENTS,
+                    "error",
+                    "No agents are available for interactive selection."
+                    " Re-run with --m365-agent-id or set M365_AGENT_ID.",
+                    operation=Operation.FETCH_AGENTS,
                 )
                 sys.exit(1)
 
             selected_agent_id, agent_name = select_agent_interactively(available_agents)
             if selected_agent_id:
-                args.m365_agent_id = selected_agent_id
-                if effective_log_level in ("info", "debug"):
-                    emit_structured_log("info", f"Selected agent: {args.m365_agent_id}", operation=Operation.FETCH_AGENTS)
+                config = replace(config, m365_agent_id=selected_agent_id)
+                if config.effective_log_level in ("info", "debug"):
+                    emit_structured_log("info", f"Selected agent: {config.m365_agent_id}", operation=Operation.FETCH_AGENTS)
             else:
                 emit_structured_log(
-                        "error",
-                        "No agent selected. Re-run with --m365-agent-id or set M365_AGENT_ID.",
-                        operation=Operation.FETCH_AGENTS,
+                    "error",
+                    "No agent selected. Re-run with --m365-agent-id or set M365_AGENT_ID.",
+                    operation=Operation.FETCH_AGENTS,
                 )
                 sys.exit(1)
     except Exception as e:
         emit_structured_log("error", f"Error during agent discovery: {e}", operation=Operation.FETCH_AGENTS)
-        if effective_log_level == "debug":
-            import traceback
+        if config.effective_log_level == "debug":
             traceback.print_exc()
         sys.exit(1)
 
-    # Pre-resolve agent endpoint (A2A agent card lookup; no-op for REST)
-    if args.m365_agent_id:
-        agent_client.resolve_agent(args.m365_agent_id)
+    # Pre-resolve agent endpoint (A2A agent card lookup)
+    if config.m365_agent_id:
+        agent_client.resolve_agent(config.m365_agent_id)
 
     # 3. Build pipeline config and run evaluation pipeline
     model_config = AzureOpenAIModelConfiguration(
-        azure_endpoint=os.environ.get("AZURE_AI_OPENAI_ENDPOINT"),
-        api_key=os.environ.get("AZURE_AI_API_KEY"),
-        api_version=os.environ.get("AZURE_AI_API_VERSION"),
-        azure_deployment=os.environ.get("AZURE_AI_MODEL_NAME"),
+        azure_endpoint=os.environ.get(ENV_AZURE_AI_OPENAI_ENDPOINT),
+        api_key=os.environ.get(ENV_AZURE_AI_API_KEY),
+        api_version=os.environ.get(ENV_AZURE_AI_API_VERSION),
+        azure_deployment=os.environ.get(ENV_AZURE_AI_MODEL_NAME),
     )
     has_azure_openai = bool(
-        os.environ.get("AZURE_AI_OPENAI_ENDPOINT")
-        and os.environ.get("AZURE_AI_API_KEY")
+        os.environ.get(ENV_AZURE_AI_OPENAI_ENDPOINT)
+        and os.environ.get(ENV_AZURE_AI_API_KEY)
     )
 
     pipeline = PipelineConfig(
@@ -1754,13 +207,14 @@ def main():
         default_evaluators=default_evaluators,
     )
 
-    results = run_pipeline(pipeline, eval_items, args)
-    
+    results = run_pipeline(pipeline, eval_items, config)
+
     # 4. Output results
-    output_results(results, args, default_evaluators=default_evaluators,
-                   agent_name=agent_name, cli_version=str(cli_version) if cli_version else None)
+    output_results(
+        results, config, default_evaluators=default_evaluators,
+        agent_name=agent_name, cli_version=str(cli_version) if cli_version else None)
     
-    if effective_log_level in ("info", "debug"):
+    if config.effective_log_level in ("info", "debug"):
         emit_structured_log(
             "info",
             f"Evaluation completed successfully. Processed {len(eval_items)} item(s).",
@@ -1768,5 +222,5 @@ def main():
         )
 
 # Call the main function when script is run directly
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     main()