@microsoft/m365-copilot-eval 1.2.1-preview.1 → 1.4.0-preview.1

package/src/clients/cli/main.py

@@ -3,14 +3,19 @@ import os
 import argparse
 import sys
 import csv
-import functools
+import logging
+import time
 import webbrowser
-import urllib.request
-import urllib.error
 import urllib.parse
 import questionary
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import List, Dict, Tuple, Optional, Any
+
+from api_clients.A2A import A2AClient
+from api_clients.REST import SydneyClient
+from api_clients.base_agent_client import BaseAgentClient
+
 from azure.ai.evaluation import (
     AzureOpenAIModelConfiguration,
     RelevanceEvaluator,
@@ -21,25 +26,119 @@ from azure.ai.evaluation import (
 from dotenv import load_dotenv
 from auth.auth_handler import AuthHandler
 from custom_evaluators.CitationsEvaluator import CitationsEvaluator, CitationFormat
-#from custom_evaluators.ConcisenessNonLLMEvaluator import ConcisenessNonLLMEvaluator
-#from custom_evaluators.PII.PII import PIIEvaluator
+from custom_evaluators.ExactMatchEvaluator import ExactMatchEvaluator
+from custom_evaluators.PartialMatchEvaluator import PartialMatchEvaluator
 from generate_report import generate_html_report, calculate_aggregate_statistics
-from response_extractor import extract_enhanced_responses, get_response_text_for_evaluation
+from response_extractor import get_response_text_for_evaluation
 from schema_handler import DocumentUpgrader, SchemaVersionManager
+from common import (
+    RELEVANCE,
+    COHERENCE,
+    GROUNDEDNESS,
+    TOOL_CALL_ACCURACY,
+    CITATIONS,
+    EXACT_MATCH,
+    PARTIAL_MATCH,
+    REQUIRES_AZURE_OPENAI,
+    REQUIRES_TOOL_DEFINITIONS,
+    METRIC_IDS,
+    STATUS_PASS,
+    STATUS_FAIL,
+    STATUS_ERROR,
+    STATUS_PARTIAL,
+    STATUS_UNKNOWN,
+    pascal_case_to_title,
+)
+from evaluator_resolver import (
+    validate_evaluator_names,
+    check_prerequisites,
+    resolve_default_evaluators,
+    resolve_evaluators_for_prompt,
+    get_evaluator_threshold,
+)
 from version_check import check_min_version, get_cli_version
 from datetime import datetime, timezone
 from pathlib import Path
-import tzlocal
+
+
+from cli_logging.console_diagnostics import emit_structured_log as _emit_structured_log
+from cli_logging.logging_utils import LOG_LEVEL_MAP, Operation, resolve_log_level
+from parallel_executor import execute_in_parallel
+from throttle_gate import ThrottleGate
+from retry_policy import (
+    is_retryable_status,
+    get_backoff_seconds,
+    get_retry_after_seconds,
+)
 
 # Allowed endpoints for URL validation
 ALLOWED_ENDPOINTS = [
-    'substrate.office.com'
+    'substrate.office.com',
+    'graph.microsoft.com',
 ]
 
+MAX_CONCURRENCY = 5
+MAX_ATTEMPTS = 4  # Initial attempt + 3 retries
+MAX_TURNS_PER_THREAD = 20
+LONG_THREAD_WARNING_THRESHOLD = 10
+
+
+@dataclass
+class PipelineConfig:
+    """Runtime configuration for the evaluation pipeline."""
+    agent_client: BaseAgentClient
+    model_config: AzureOpenAIModelConfiguration
+    has_azure_openai: bool
+    default_evaluators: Dict[str, Any]
+    chat_gate: ThrottleGate = field(default_factory=lambda: ThrottleGate("chat_api"))
+    is_retryable_status: Any = field(default=is_retryable_status)
+    get_backoff_seconds: Any = field(default=get_backoff_seconds)
+    get_retry_after_seconds: Any = field(default=get_retry_after_seconds)
+
 class CallPath(Enum):
     """ Enum to indicate which call path to use. """
     ACCESS_TOKEN = "access_token"
     COPILOT_AUTH = "copilot_auth"
+    A2A = "a2a"
+
+
+class ItemType(Enum):
+    SINGLE_TURN = "single_turn"
+    MULTI_TURN = "multi_turn"
+
+
+def detect_item_type(item: dict) -> ItemType:
+    """Determine if an evaluation item is single-turn or multi-turn.
+
+    Returns ItemType.SINGLE_TURN if item has 'prompt' without 'turns',
+    ItemType.MULTI_TURN if item has 'turns' array.
+
+    Raises ValueError for invalid items (both, neither, or invalid turns).
+    """
+    has_turns = "turns" in item
+    has_prompt = "prompt" in item
+
+    if has_turns and has_prompt:
+        raise ValueError(
+            "Invalid evaluation item: cannot have both 'turns' and 'prompt'. "
+            "Use 'turns' for multi-turn threads or 'prompt' for single-turn."
+        )
+
+    if has_turns and not isinstance(item["turns"], list):
+        raise ValueError("Invalid evaluation item: 'turns' must be a list")
+
+    if has_turns:
+        if len(item["turns"]) == 0:
+            raise ValueError("Invalid multi-turn thread: 'turns' array cannot be empty")
+        return ItemType.MULTI_TURN
+
+    if has_prompt:
+        return ItemType.SINGLE_TURN
+
+    raise ValueError(
+        "Invalid evaluation item: must have either 'turns' array (multi-turn) "
+        "or 'prompt' field (single-turn)"
+    )
 
 
 # Flags that should bypass remote min-version enforcement.
@@ -48,20 +147,44 @@ VERSION_CHECK_BYPASS_FLAGS = (
     "signout",
 )
 
+CLI_LOGGER_NAME = "m365.eval.cli"
+CLI_LOGGER = logging.getLogger(CLI_LOGGER_NAME)
+DIAGNOSTIC_RECORDS: List[Dict[str, Any]] = []
+
+
+def configure_cli_logging(effective_log_level: str) -> None:
+    if not CLI_LOGGER.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        CLI_LOGGER.addHandler(handler)
+        CLI_LOGGER.propagate = False
+    CLI_LOGGER.setLevel(LOG_LEVEL_MAP[effective_log_level])
+
+
+def emit_structured_log(level: str, message: str, operation: str = Operation.EVALUATE) -> None:
+    _emit_structured_log(
+        level, message, operation,
+        logger=CLI_LOGGER,
+        diagnostic_records=DIAGNOSTIC_RECORDS,
+    )
+
 
 def should_bypass_min_version_check(args: argparse.Namespace) -> bool:
     """Return True if the current invocation should skip min-version checks."""
     return any(getattr(args, flag, False) for flag in VERSION_CHECK_BYPASS_FLAGS)
 
-def write_results_to_html(results: List[Dict], output_file: str):
+def write_results_to_html(results: List[Dict], output_file: str,
+                          agent_name: Optional[str] = None, agent_id: Optional[str] = None,
+                          cli_version: Optional[str] = None):
     """Write results to HTML file using generate_html_report from generate_report.py."""
     try:
-        html = generate_html_report(results) 
+        html = generate_html_report(results, agent_name=agent_name, agent_id=agent_id,
+                                    cli_version=cli_version)
         with open(output_file, 'w', encoding='utf-8') as f:
             f.write(html)
-        print(f"HTML report saved to {output_file}")
+        emit_structured_log("info", f"HTML report saved to {output_file}", operation=Operation.WRITE_OUTPUT)
     except Exception as e:
-        print(f"Error writing to HTML file: {e}")
+        emit_structured_log("error", f"Error writing to HTML file: {e}", operation=Operation.WRITE_OUTPUT)
         sys.exit(1)
 
 def get_default_prompts_and_responses():
@@ -74,7 +197,7 @@ def get_default_prompts_and_responses():
     ]
     return prompts, expected_responses
 
-def load_prompts_from_file(file_path: str) -> Tuple[List[str], List[str]]:
+def load_prompts_from_file(file_path: str) -> Tuple[List[Dict], Optional[Dict]]:
     """Load prompts and expected responses from a JSON file.
 
     Supports three formats:
@@ -84,6 +207,10 @@ def load_prompts_from_file(file_path: str) -> Tuple[List[str], List[str]]:
 
     For eval documents (format 1) and array format (format 2), schema validation
     and auto-upgrade are applied via DocumentUpgrader.
+
+    Returns:
+        Tuple of (eval_items, default_evaluators). Items are dicts with prompt,
+        expected_response, and optional evaluators/evaluators_mode fields.
     """
     try:
         with open(file_path, 'r', encoding='utf-8') as f:
@@ -100,18 +227,18 @@ def load_prompts_from_file(file_path: str) -> Tuple[List[str], List[str]]:
                 upgrader = DocumentUpgrader()
             except Exception as e:
                 # Schema infrastructure not available (missing files, etc.) — skip
-                print(f"Warning: Unable to initialize document upgrader: {e}")
+                emit_structured_log("warning", f"Unable to initialize document upgrader: {e}", operation=Operation.LOAD_PROMPTS)
                 upgrader = None
 
             if upgrader is not None:
                 result = upgrader.upgrade(Path(file_path))
 
                 if result.error:
-                    print(f"Schema validation error: {result.error}")
+                    emit_structured_log("error", f"Schema validation error: {result.error}", operation=Operation.LOAD_PROMPTS)
                     sys.exit(1)
 
                 if result.upgraded and result.message:
-                    print(result.message)
+                    emit_structured_log("info", result.message, operation=Operation.LOAD_PROMPTS)
 
                 # Use the parsed document from the upgrade result
                 if result.document is not None:
@@ -119,26 +246,26 @@ def load_prompts_from_file(file_path: str) -> Tuple[List[str], List[str]]:
 
         if isinstance(data, list):
             # Format: [{"prompt": "...", "expected_response": "..."}, ...]
-            prompts = [item.get("prompt", "") for item in data]
-            expected_responses = [item.get("expected_response", "") for item in data]
+            return data, None
         elif isinstance(data, dict):
             if "items" in data:
                 # Eval document format: {"schemaVersion": "...", "items": [...]}
-                items = data["items"]
-                prompts = [item.get("prompt", "") for item in items]
-                expected_responses = [item.get("expected_response", "") for item in items]
+                return data["items"], data.get("default_evaluators")
             else:
                 # Format: {"prompts": [...], "expected_responses": [...]}
                 prompts = data.get("prompts", [])
                 expected_responses = data.get("expected_responses", [])
+                eval_items = [
+                    {"prompt": p, "expected_response": e}
+                    for p, e in zip(prompts, expected_responses)
+                ]
+                return eval_items, None
         else:
             raise ValueError("Invalid file format")
-
-        return prompts, expected_responses
     except SystemExit:
         raise
     except Exception as e:
-        print(f"Error loading prompts from file: {e}")
+        emit_structured_log("error", f"Error loading prompts from file: {e}", operation=Operation.LOAD_PROMPTS)
         sys.exit(1)
 
 def get_interactive_prompts() -> Tuple[List[str], List[str]]:
@@ -165,116 +292,549 @@ def get_interactive_prompts() -> Tuple[List[str], List[str]]:
     
     return prompts, expected_responses
 
-def run_evaluations(args, responses: dict, expected_responses: list) -> list:
-    """Run evaluations against the responses."""
-    model_config = AzureOpenAIModelConfiguration(
-        azure_endpoint=os.environ.get("AZURE_AI_OPENAI_ENDPOINT"),
-        api_key=os.environ.get("AZURE_AI_API_KEY"),
-        api_version=os.environ.get("AZURE_AI_API_VERSION"),
-        azure_deployment=os.environ.get("AZURE_AI_MODEL_NAME"),
+
+_DEFAULT_PASS_THRESHOLD = 3
+
+
+def _decorate_metric(metric_id: str, data, threshold: Optional[int] = None) -> Dict[str, Any]:
+    """Augment raw evaluator output with standardized threshold + pass/fail result."""
+    pass_threshold = threshold if threshold is not None else _DEFAULT_PASS_THRESHOLD
+    payload = {}
+    if isinstance(data, dict):
+        payload.update(data)
+    else:
+        payload['raw'] = data
+
+    score_val = None
+    if isinstance(data, dict):
+        if metric_id in data:
+            score_val = data[metric_id]
+    if isinstance(score_val, (int, float)):
+        payload['threshold'] = pass_threshold
+        payload['result'] = STATUS_PASS if score_val >= pass_threshold else STATUS_FAIL
+    else:
+        payload['threshold'] = pass_threshold
+        payload.setdefault('result', STATUS_UNKNOWN)
+    return payload
+
+
+def _run_evaluators_for_item(
+    prompt: str,
+    actual_response: str,
+    expected_response: str,
+    enhanced_response: Dict[str, Any],
+    resolved_evaluators: Dict[str, Any],
+    model_config: AzureOpenAIModelConfiguration,
+    has_azure_openai: bool,
+    args,
+) -> Tuple[Dict[str, Optional[str]], List[str]]:
+    """Run resolved evaluators against a single item/turn.
+
+    Returns (results_dict, evaluators_ran).
+    """
+    has_tool_defs = bool(
+        args.m365_agent_id and enhanced_response.get("tool_definitions")
     )
-    
-    # Initialize evaluators
-    relevance_evaluator = RelevanceEvaluator(model_config=model_config) # Evaluate relevance for a given response. Range is 1 - 5.
-    coherence_evaluator = CoherenceEvaluator(model_config=model_config) # Measures the coherence (human-like quality) of the response. Range is 1 - 5.
-    groundedness_evaluator = GroundednessEvaluator(model_config=model_config) # Evaluates the response for for factuality and groundedness against provided context. Range is 1 - 5.
-    #concisenessnonllm_evaluator = ConcisenessNonLLMEvaluator() # Evaluates the response for conciseness. Range is 1 - 5.
-    #pii_evaluator = PIIEvaluator(model_config=model_config) # Evaluates the response for presence of PII. Range
-    # Parse citation format from args
-    citation_format = CitationFormat.OAI_UNICODE if args.citation_format == 'oai_unicode' else CitationFormat.LEGACY_BRACKET
-    citations_evaluator = CitationsEvaluator(citation_format=citation_format) # Evaluates citations present in the response using regex pattern matching
-
-    tool_call_accuracy_evaluator = ToolCallAccuracyEvaluator(model_config) # Evaluate tool call accuracy if tool definitions are present in response
-    
+    available_context = {
+        REQUIRES_AZURE_OPENAI: has_azure_openai,
+        REQUIRES_TOOL_DEFINITIONS: has_tool_defs,
+    }
 
-    PASS_THRESHOLD = 3  # All evaluators must meet or exceed this value (out of 5) to pass
+    results_dict: Dict[str, Optional[str]] = {}
+    evaluators_ran: List[str] = []
+
+    for eval_name, eval_options in resolved_evaluators.items():
+        can_run, warn_msg = check_prerequisites(eval_name, available_context)
+        if not can_run:
+            if warn_msg:
+                emit_structured_log(
+                    "warning",
+                    f"Evaluator '{eval_name}' prerequisite check failed: {warn_msg}",
+                    operation=Operation.EVALUATE,
+                )
+            results_dict[eval_name] = None
+            continue
 
-    def decorate_metric(metric_id: str, data):
-        """Augment raw evaluator output with standardized threshold + pass/fail result."""
-        payload = {}
-        # Preserve original structure if dict
-        if isinstance(data, dict):
-            payload.update(data)
-        else:
-            payload['raw'] = data
-
-        # Try to extract a numeric score
-        score_val = None
-        if isinstance(data, dict):
-            for k in (metric_id, f"{metric_id}_score", 'score', 'value'):
-                if k in data:
-                    score_val = data[k]
-                    break
-        if isinstance(score_val, (int, float)):
-            payload['threshold'] = PASS_THRESHOLD
-            payload['result'] = 'pass' if score_val >= PASS_THRESHOLD else 'fail'
-        else:
-            # If we cannot determine score, mark unknown (no pass/fail)
-            payload['threshold'] = PASS_THRESHOLD
-            payload.setdefault('result', 'unknown')
-        return json.dumps(payload, indent=4)
-
-    evaluation_results = []
-    for prompt, expected_response in zip(responses.keys(), expected_responses):
-        # Extract text response for evaluation (backward compatibility)
-        enhanced_response = responses[prompt]
-        actual_response_text = get_response_text_for_evaluation(enhanced_response)
-        
-        # Run evaluations using text response
-        relevance_score = relevance_evaluator(
-            query=prompt,
-            response=actual_response_text
+        threshold = get_evaluator_threshold(eval_name, eval_options)
+
+        try:
+            if eval_name == RELEVANCE:
+                raw_score = RelevanceEvaluator(model_config=model_config)(query=prompt, response=actual_response)
+                results_dict[RELEVANCE] = _decorate_metric(METRIC_IDS[RELEVANCE], raw_score, threshold)
+            elif eval_name == COHERENCE:
+                raw_score = CoherenceEvaluator(model_config=model_config)(query=prompt, response=actual_response)
+                results_dict[COHERENCE] = _decorate_metric(METRIC_IDS[COHERENCE], raw_score, threshold)
+            elif eval_name == GROUNDEDNESS:
+                raw_score = GroundednessEvaluator(model_config=model_config)(response=actual_response, context=expected_response)
+                results_dict[GROUNDEDNESS] = _decorate_metric(METRIC_IDS[GROUNDEDNESS], raw_score, threshold)
+            elif eval_name == TOOL_CALL_ACCURACY:
+                raw_score = ToolCallAccuracyEvaluator(model_config)(
+                    query=prompt,
+                    response=enhanced_response.get("response", actual_response),
+                    tool_definitions=enhanced_response.get("tool_definitions", []),
+                )
+                results_dict[TOOL_CALL_ACCURACY] = _decorate_metric(METRIC_IDS[TOOL_CALL_ACCURACY], raw_score, threshold)
+            elif eval_name == CITATIONS:
+                fmt_str = eval_options.get("citation_format", "oai_unicode")
+                fmt_map = {
+                    "oai_unicode": CitationFormat.OAI_UNICODE,
+                    "bracket": CitationFormat.LEGACY_BRACKET,
+                    "mixed": CitationFormat.AUTO,
+                }
+                raw_score = CitationsEvaluator(citation_format=fmt_map.get(fmt_str, CitationFormat.OAI_UNICODE))(response=actual_response)
+                results_dict[CITATIONS] = _decorate_metric(METRIC_IDS[CITATIONS], raw_score, threshold)
+            elif eval_name == EXACT_MATCH:
+                case_sensitive = eval_options.get("case_sensitive", False)
+                raw_score = ExactMatchEvaluator(case_sensitive=case_sensitive)(response=actual_response, expected_answer=expected_response)
+                # ExactMatch is binary — the evaluator already sets 'result'
+                # so _decorate_metric (which computes result from score vs threshold) is not needed.
+                results_dict[EXACT_MATCH] = raw_score
+            elif eval_name == PARTIAL_MATCH:
+                case_sensitive = eval_options.get("case_sensitive", False)
+                raw_score = PartialMatchEvaluator(case_sensitive=case_sensitive)(response=actual_response, expected_answer=expected_response)
+                results_dict[PARTIAL_MATCH] = _decorate_metric(METRIC_IDS[PARTIAL_MATCH], raw_score, threshold)
+
+            evaluators_ran.append(eval_name)
+        except Exception as e:
+            emit_structured_log(
+                "error",
+                f"Evaluator '{eval_name}' crashed and will be omitted from results: {e}",
+                operation=Operation.EVALUATE,
+            )
+            results_dict[eval_name] = None
+
+    return results_dict, evaluators_ran
+
+
+def _evaluate_single_response(
+    enhanced_response: Dict[str, Any],
+    eval_item: Dict,
+    args,
+    model_config: AzureOpenAIModelConfiguration,
+    has_azure_openai: bool,
+    default_evaluators: Dict[str, Any],
+) -> Dict[str, Any]:
+    """Run all evaluators for a single prompt/response pair and return the result dict."""
+    actual_response_text = get_response_text_for_evaluation(enhanced_response)
+    prompt = eval_item.get("prompt", "")
+    expected_response = eval_item.get("expected_response", "")
+
+    resolved = resolve_evaluators_for_prompt(
+        eval_item.get("evaluators"), eval_item.get("evaluators_mode", "extend"),
+        prompt, default_evaluators,
+    )
+
+    results_dict, evaluators_ran = _run_evaluators_for_item(
+        prompt, actual_response_text, expected_response, enhanced_response,
+        resolved, model_config, has_azure_openai, args,
+    )
+
+    evaluation_result = {
+        "prompt": prompt,
+        "response": enhanced_response.get(
+            "display_response_text", actual_response_text
+        ),
+        "expected_response": expected_response,
+        "evaluators_ran": evaluators_ran,
+        "results": results_dict,
+    }
+
+    if "evaluators" in eval_item:
+        evaluation_result["evaluators"] = eval_item["evaluators"]
+    if "evaluators_mode" in eval_item:
+        evaluation_result["evaluators_mode"] = eval_item["evaluators_mode"]
+
+    if getattr(args, "effective_log_level", "info") == "debug":
+        emit_structured_log(
+            "debug",
+            f"Evaluation completed for prompt='{evaluation_result['prompt']}'. "
+            f"Evaluators: {', '.join(evaluators_ran)}. "
+            f"Scores: {evaluation_result['results']}",
+            operation=Operation.EVALUATE,
+        )
+
+    return evaluation_result
+
+
+def _check_all_passed(results_dict: Dict[str, Optional[Dict[str, Any]]]) -> bool:
+    """Check if all evaluator results passed. Skipped evaluators (None) are ignored."""
+    for result_data in results_dict.values():
+        if result_data is None:
+            continue
+        if result_data.get("result") == STATUS_FAIL:
+            return False
+    return True
+
+def _evaluate_multi_turn_responses(
+    turns: List[Dict],
+    args,
+    default_evaluators: Dict[str, Any],
+    model_config: AzureOpenAIModelConfiguration,
+    has_azure_openai: bool,
+) -> Tuple[List[Dict], Dict]:
+    """Run per-turn evaluations and build evaluated turn results with summary.
+
+    Returns:
+        Tuple of (evaluated_turns, summary). Each evaluated turn contains
+        prompt, response, expected_response, status, evaluators_ran, results,
+        and optionally error. Does not mutate the input turns.
+    """
+    evaluated_turns: List[Dict] = []
+    turns_passed = 0
+    turns_failed = 0
+
+    for i, turn in enumerate(turns):
+        evaluated_turn: Dict[str, Any] = {
+            "prompt": turn.get("prompt", ""),
+        }
+        if "expected_response" in turn:
+            evaluated_turn["expected_response"] = turn["expected_response"]
+        if "response" in turn:
+            evaluated_turn["response"] = turn["response"]
+        if "evaluators" in turn:
+            evaluated_turn["evaluators"] = turn["evaluators"]
+        if "evaluators_mode" in turn:
+            evaluated_turn["evaluators_mode"] = turn["evaluators_mode"]
+
+        if turn.get("status") == STATUS_ERROR:
+            evaluated_turn["status"] = STATUS_ERROR
+            evaluated_turn["error"] = turn.get("error", "")
+            turns_failed += 1
+            evaluated_turns.append(evaluated_turn)
+            continue
+
+        enhanced_response = turn.get("_enhanced_response", {})
+        actual_response = get_response_text_for_evaluation(enhanced_response)
+
+        resolved = resolve_evaluators_for_prompt(
+            turn.get("evaluators"), turn.get("evaluators_mode", "extend"),
+            turn.get("prompt", ""), default_evaluators,
         )
-        coherence_score = coherence_evaluator(
-            query=prompt,
-            response=actual_response_text
+
+        results_dict, evaluators_ran = _run_evaluators_for_item(
+            turn.get("prompt", ""), actual_response, turn.get("expected_response", ""),
+            enhanced_response, resolved, model_config, has_azure_openai, args,
         )
 
-        groundedness_score = groundedness_evaluator(
-            response=actual_response_text,
-            context=expected_response
+        all_passed = _check_all_passed(results_dict)
+
+        evaluated_turn["results"] = results_dict
+        evaluated_turn["evaluators_ran"] = evaluators_ran
+        evaluated_turn["status"] = STATUS_PASS if all_passed else STATUS_FAIL
+
+        if getattr(args, "effective_log_level", "info") == "debug":
+            emit_structured_log(
+                "debug",
+                f"Evaluation completed for turn {i + 1} prompt='{turn.get('prompt', '')}'. "
+                f"Evaluators: {', '.join(evaluators_ran)}. "
+                f"Scores: {results_dict}",
+                operation=Operation.EVALUATE,
+            )
+
+        if all_passed:
+            turns_passed += 1
+        else:
+            turns_failed += 1
+
+        evaluated_turns.append(evaluated_turn)
+
+    turns_total = len(turns)
+    if turns_passed == turns_total:
+        overall_status = STATUS_PASS
+    elif turns_failed == turns_total:
+        overall_status = STATUS_FAIL
+    else:
+        overall_status = STATUS_PARTIAL
+
+    summary = {
+        "turns_total": turns_total,
+        "turns_passed": turns_passed,
+        "turns_failed": turns_failed,
+        "overall_status": overall_status,
+    }
+
+    return evaluated_turns, summary
+
+
+def get_effective_worker_count(prompt_count: int, args) -> int:
+    """Compute safe worker count for prompt processing."""
+    if prompt_count <= 0:
+        return 1
+
+    requested = getattr(args, "concurrency", 5)
+    try:
+        requested_int = int(requested)
+    except (TypeError, ValueError):
+        requested_int = 5
+
+    bounded = max(1, min(requested_int, MAX_CONCURRENCY))
+    return min(bounded, prompt_count)
+
+
+def run_pipeline(
+    pipeline: PipelineConfig,
+    eval_items: List[Dict],
+    args,
+) -> List[Dict[str, Any]]:
+    """Run the full evaluation pipeline: send prompts and evaluate responses in parallel.
+
+    Each worker processes one prompt end-to-end: send → evaluate.
+    Results are returned in original prompt order (FR-006).
+    """
+    # Validate all evaluator names upfront before dispatching workers
+    all_evaluator_maps = [pipeline.default_evaluators]
+    for eval_item in eval_items:
+        if "evaluators" in eval_item:
+            all_evaluator_maps.append(eval_item["evaluators"])
+        for turn in eval_item.get("turns", []):
+            if "evaluators" in turn:
+                all_evaluator_maps.append(turn["evaluators"])
+    for emap in all_evaluator_maps:
+        validate_evaluator_names(emap)
+
+    # Validate all items upfront and classify types before dispatching workers
+    item_types: List[ItemType] = []
+    for idx, eval_item in enumerate(eval_items):
+        try:
+            item_type = detect_item_type(eval_item)
+        except ValueError as e:
+            raise ValueError(f"Invalid evaluation item at index {idx}: {e}") from e
+        if item_type == ItemType.MULTI_TURN:
+            turn_count = len(eval_item["turns"])
+            if turn_count > MAX_TURNS_PER_THREAD:
+                raise ValueError(
+                    f"Invalid evaluation item at index {idx}: 'turns' array has "
+                    f"{turn_count} items (max {MAX_TURNS_PER_THREAD})"
+                )
+        item_types.append(item_type)
+
+    total = len(eval_items)
+    worker_count = get_effective_worker_count(total, args)
+
+    multi_turn_count = sum(1 for t in item_types if t == ItemType.MULTI_TURN)
+    single_turn_count = total - multi_turn_count
+
+    emit_structured_log(
+        "info",
+        f"Running pipeline with {worker_count} worker(s) for {total} item(s) "
+        f"({single_turn_count} single-turn, {multi_turn_count} multi-turn).",
+        operation=Operation.EVALUATE,
+    )
+
+    def _process_item(eval_item: Dict, index: int) -> Dict[str, Any]:
+        if item_types[index] == ItemType.MULTI_TURN:
+            return _process_multi_turn(eval_item, index)
+        return _process_single_turn(eval_item, index)
+
+    def _process_single_turn(eval_item: Dict, index: int) -> Dict[str, Any]:
+        prompt = eval_item.get("prompt", "")
+        emit_structured_log(
+            "info",
+            f"Processing item {index + 1}/{total} (single-turn).",
+            operation=Operation.SEND_PROMPT,
         )
 
-        #PII_score = pii_evaluator(response=actual_response_text)
-        #concisenessNonLLM_score = concisenessnonllm_evaluator(response=actual_response_text)
+        # Phase A: Send prompt to agent (with retry + throttle gate)
+        response = None
+        for attempt in range(1, MAX_ATTEMPTS + 1):
+            pipeline.chat_gate.wait_if_blocked()
+            try:
+                response, _ = pipeline.agent_client.send_prompt(prompt, agent_id=args.m365_agent_id)
+                break
+            except Exception as exc:
+                cause = exc.__cause__
+                status = int(getattr(cause, "code", 0) or 0) or None if cause else None
+                retry_after = get_retry_after_seconds(
+                    cause.headers.get("Retry-After") if cause and getattr(cause, "headers", None) else None
+                )
+
+                if retry_after is not None and pipeline.is_retryable_status(status):
+                    pipeline.chat_gate.apply_retry_after(retry_after)
+
+                if not pipeline.is_retryable_status(status) or attempt >= MAX_ATTEMPTS:
+                    emit_structured_log(
+                        "error",
+                        f"Item {index + 1}/{total} failed after {attempt} attempt(s): {exc}",
+                        operation=Operation.SEND_PROMPT,
+                    )
+                    return {
+                        "prompt": prompt,
+                        "response": "",
+                        "expected_response": eval_item.get("expected_response", ""),
+                        "evaluators_ran": [],
+                        "results": {},
+                        "status": STATUS_ERROR,
+                        "errorDetails": str(exc),
+                    }
+
+                delay = retry_after if retry_after is not None else pipeline.get_backoff_seconds(attempt)
+                time.sleep(delay)
+
+        # Phase B: Evaluate response
+        return _evaluate_single_response(
+            response, eval_item, args,
+            pipeline.model_config, pipeline.has_azure_openai,
+            pipeline.default_evaluators,
+        )
 
-        citations_score = citations_evaluator(
-            response=actual_response_text
+    def _process_multi_turn(eval_item: Dict, index: int) -> Dict[str, Any]:
+        turns = eval_item["turns"]
+        thread_name = eval_item.get("name", "Unnamed thread")
+        emit_structured_log(
+            "info",
+            f"Processing item {index + 1}/{total} (multi-turn: '{thread_name}').",
+            operation=Operation.SEND_PROMPT,
         )
 
-        tool_call_accuracy = None
-        if args.m365_agent_id and enhanced_response.get("tool_definitions"):
-            tool_call_accuracy = tool_call_accuracy_evaluator(
-                query=prompt,
-                response=enhanced_response.get("response", actual_response_text),
-                tool_definitions=enhanced_response["tool_definitions"]
+        if len(turns) > LONG_THREAD_WARNING_THRESHOLD:
+            emit_structured_log(
+                "warning",
+                f"Thread '{thread_name}' has {len(turns)} turns (>{LONG_THREAD_WARNING_THRESHOLD}). This may take a while.",
+                operation=Operation.SEND_PROMPT,
             )
 
-        evaluation_result = {
-            "prompt": prompt,
-            "response": actual_response_text,  # Keep simple text for backward compatibility
-            "expected_response": expected_response,
-            "results": {
-                "relevance_score": decorate_metric("relevance", relevance_score),
-                "coherence_score": decorate_metric("coherence", coherence_score),
-                "groundedness_score": decorate_metric("groundedness", groundedness_score),
-                #"concisenessnonllm_score": decorate_metric("concisenessnonllm", concisenessNonLLM_score),
-                #"pii_score": decorate_metric("pii", PII_score),
-                "citations_score": json.dumps(citations_score, indent=4),
-                "tool_call_accuracy_score": json.dumps(tool_call_accuracy, indent=4) if tool_call_accuracy else None
-            }
+        # Phase A: Send each turn with throttle gate + 429-only retry
+        # Multi-turn only retries on 429 (server confirmed it didn't process
+        # the request). Other transient errors (503, 504) are ambiguous about
+        # whether the server processed the turn, risking duplicate turns in
+        # the conversation if retried.
+        conversation_context = None
+        conversation_id = None
+        enriched_turns: List[Dict[str, Any]] = []
+        failed = False
+
+        for i, turn in enumerate(turns):
+            prompt = turn["prompt"]
+            emit_structured_log(
+                "debug",
+                f"Sending turn {i + 1}/{len(turns)} of '{thread_name}'.",
+                operation=Operation.SEND_PROMPT,
+            )
+
+            response = None
+            for attempt in range(1, MAX_ATTEMPTS + 1):
+                pipeline.chat_gate.wait_if_blocked()
+                try:
+                    response, conversation_context = pipeline.agent_client.send_prompt(
+                        prompt, agent_id=args.m365_agent_id,
+                        conversation_context=conversation_context,
+                    )
+                    break
+                except Exception as exc:
+                    cause = exc.__cause__
+                    status = int(getattr(cause, "code", 0) or 0) or None if cause else None
+                    retry_after = get_retry_after_seconds(
+                        cause.headers.get("Retry-After") if cause and getattr(cause, "headers", None) else None
+                    )
+
+                    # Only retry on 429 — server confirmed it didn't process the request
+                    if status == 429 and attempt < MAX_ATTEMPTS:
+                        if retry_after is not None:
+                            pipeline.chat_gate.apply_retry_after(retry_after)
+                        delay = retry_after if retry_after is not None else pipeline.get_backoff_seconds(attempt)
+                        time.sleep(delay)
+                        continue
+
+                    # All other errors: stop the thread
+                    emit_structured_log(
+                        "error",
+                        f"Turn {i + 1}/{len(turns)} of '{thread_name}' failed after {attempt} attempt(s): {exc}",
+                        operation=Operation.SEND_PROMPT,
+                    )
+                    failed = True
+                    break
+
+            if failed:
+                # Mark this turn and all remaining turns as error
+                enriched_turns.append({
+                    **turn,
+                    "response": "",
+                    "status": STATUS_ERROR,
+                    "error": "Failed to get response from agent",
+                })
+                for j in range(i + 1, len(turns)):
+                    enriched_turns.append({
+                        **turns[j],
+                        "response": "",
+                        "status": STATUS_ERROR,
+                        "error": "Skipped: preceding turn failed",
+                    })
+                break
+
+            # Enrich turn with response
+            response_text = get_response_text_for_evaluation(response)
+            enriched_turns.append({
+                **turn,
+                "response": response.get("display_response_text", response_text),
+                "_enhanced_response": response,
+            })
+
+            # Capture conversation_id from first response
+            if conversation_id is None:
+                conversation_id = response.get("metadata", {}).get("conversation_id")
+
+        # Phase B: Run per-turn evaluations
+        evaluated_turns, summary = _evaluate_multi_turn_responses(
+            enriched_turns, args, pipeline.default_evaluators,
+            model_config=pipeline.model_config,
+            has_azure_openai=pipeline.has_azure_openai,
+        )
+
+        return {
+            "type": "multi_turn",
+            "name": eval_item.get("name", ""),
+            "description": eval_item.get("description", ""),
+            "conversation_id": conversation_id or "",
+            "turns": evaluated_turns,
+            "summary": summary,
         }
 
-        if args.verbose:
-            print(f".................................. Evaluation for prompt: {evaluation_result['prompt']} ..................................")
-            print(f"Scores: {evaluation_result['results']}")
-            print("...........................................................................................................................")
+    execution_results = execute_in_parallel(
+        eval_items, _process_item, max_workers=worker_count,
+    )
+
+    # Unwrap WorkerResult objects into plain dicts, with error fallback
+    ordered_results: List[Dict[str, Any]] = []
+    for wr in execution_results:
+        if wr.error:
+            idx = wr.index
+            item = eval_items[idx]
+            if item_types[idx] == ItemType.MULTI_TURN:
+                ordered_results.append({
+                    "type": "multi_turn",
+                    "name": item.get("name", ""),
+                    "turns": [
+                        {**t, "status": STATUS_ERROR, "error": str(wr.error), "response": "", "results": {}}
+                        for t in item.get("turns", [])
+                    ],
+                    "summary": {
+                        "turns_total": len(item.get("turns", [])),
+                        "turns_passed": 0,
+                        "turns_failed": len(item.get("turns", [])),
+                        "overall_status": STATUS_FAIL,
+                    },
+                    "error": str(wr.error),
+                })
+            else:
+                ordered_results.append({
+                    "prompt": item.get("prompt", ""),
+                    "response": "",
+                    "expected_response": item.get("expected_response", ""),
+                    "evaluators_ran": [],
+                    "results": {},
+                    "status": STATUS_ERROR,
+                    "errorDetails": str(wr.error),
+                })
+        else:
+            ordered_results.append(wr.value)
+
+    return ordered_results
 
-        evaluation_results.append(evaluation_result)
-    
-    return evaluation_results
 
-def write_results_to_console(results):
+
+def write_results_to_console(results, agent_name: Optional[str] = None,
+                             agent_id: Optional[str] = None,
+                             cli_version: Optional[str] = None):
     """Write the response to console."""
     # ANSI color codes
     BOLD = '\033[1m'
@@ -286,48 +846,105 @@ def write_results_to_console(results):
     ORANGE = '\033[38;5;208m'
     RED = '\033[91m'
     RESET = '\033[0m'
-    
-    # Show aggregate statistics if multiple results
-    if len(results) > 1:
-        aggregates = calculate_aggregate_statistics(results)
-        if aggregates:
-            print(f"{BOLD}{BLUE}📊 Aggregate Statistics ({len(results)} prompts):{RESET}")
+
+    def _print_evaluated_item(response: str, expected_response: str,
+                              evaluators_ran: List[str], item_results: Dict[str, Any],
+                              error: Optional[str] = None) -> None:
+        """Print the body of a single evaluated item (single-turn prompt or multi-turn turn).
+
+        The item header (Prompt X / Turn X) is printed by the caller; this helper
+        prints evaluators, response, expected response, error, and metrics.
+        """
+        if evaluators_ran:
+            print(f"{BOLD}{CYAN}Evaluators:{RESET} {', '.join(evaluators_ran)}")
+        if response:
+            print(f"{BOLD}{CYAN}Response:{RESET} {response}")
+        if expected_response:
+            print(f"{BOLD}{YELLOW}Expected Response:{RESET} {expected_response}")
+        if error:
+            print(f"{BOLD}{RED}Error:{RESET} {error}")
+
+        for eval_name, v in item_results.items():
+            if v is None:
+                continue
+            display_name = pascal_case_to_title(eval_name)
+            if eval_name == RELEVANCE:
+                color = MAGENTA
+            elif eval_name == COHERENCE:
+                color = ORANGE
+            else:
+                color = BLUE
+            print(f"{BOLD}{color}{display_name}:{RESET} {json.dumps(v, indent=4)}")
+
+    # Show metadata
+    metadata_parts = []
+    if agent_name:
+        metadata_parts.append(f"Agent Name: {agent_name}")
+    if agent_id:
+        metadata_parts.append(f"Agent ID: {agent_id}")
+    if cli_version:
+        metadata_parts.append(f"CLI Version: {cli_version}")
+    if metadata_parts:
+        print(f"{BOLD}{CYAN}{' | '.join(metadata_parts)}{RESET}")
+        print()
+
+    aggregates = calculate_aggregate_statistics(results)
+    if aggregates:
+        total_items = aggregates[next(iter(aggregates))].get('total_prompts', len(results))
+        if total_items > 1:
+            print(f"{BOLD}{BLUE}Aggregate Statistics ({total_items} prompts):{RESET}")
             print(f"{BLUE}{'=' * 60}{RESET}")
-            
+
             for metric_name, stats in aggregates.items():
                 pass_color = GREEN if stats['pass_rate'] >= 80 else YELLOW if stats['pass_rate'] >= 60 else RED
-                print(f"{BOLD}{CYAN}{metric_name}:{RESET}")
+                prompts_evaluated = stats.get('prompts_evaluated', stats['total_evaluated'])
+                total_prompts = stats.get('total_prompts', total_items)
+                print(f"{BOLD}{CYAN}{metric_name}:{RESET} ({prompts_evaluated}/{total_prompts} prompts)")
                 print(f"  Pass Rate: {pass_color}{stats['pass_rate']:.1f}%{RESET} ({stats['pass_count']}/{stats['total_evaluated']} passed)")
                 print(f"  Avg Score: {MAGENTA}{stats['avg_score']:.2f}{RESET}")
                 if stats.get('threshold') is not None:
                     print(f"  Threshold: {YELLOW}{stats['threshold']}{RESET}")
                 print()
-            
+
             print(f"{BLUE}{'=' * 60}{RESET}")
             print()
-    
-    print(f"{BOLD}{BLUE}📝 Individual Results:{RESET}")
+
+    print(f"{BOLD}{BLUE}Individual Results:{RESET}")
     print(f"{BLUE}{'=' * 50}{RESET}")
     for i, result in enumerate(results, 1):
-        print(f"{BOLD}{GREEN}Prompt {i}:{RESET} {result['prompt']}")
-        print(f"{BOLD}{CYAN}Response:{RESET} {result['response']}")
-        print(f"{BOLD}{YELLOW}Expected Response:{RESET} {result['expected_response']}")
-
-        # Print metric scores generically from nested results (fallback to flat keys for back-compat)
-        metrics = result.get('results') or {k: v for k, v in result.items() if isinstance(k, str) and k.endswith('_score')}
-        if metrics:
-            for k, v in metrics.items():
-                name = k.replace('_', ' ')
-                if 'relevance' in k:
-                    color = MAGENTA
-                elif 'coherence' in k:
-                    color = ORANGE
-                elif 'fluency' in k:
-                    color = GREEN
-                else:
-                    color = BLUE
-                print(f"{BOLD}{color}{name}:{RESET} {v}")
-        print(f"{BLUE}{'-' * 30}{RESET}")
+        if result.get("type") == "multi_turn":
+            thread_name = result.get("name", "Unnamed Thread")
+            summary = result.get("summary", {})
+            status = summary.get("overall_status", STATUS_UNKNOWN)
+            status_color = GREEN if status == STATUS_PASS else YELLOW if status == STATUS_PARTIAL else RED
+
+            print(f"{BOLD}{MAGENTA}Thread {i}: {thread_name}{RESET}")
+            for t_idx, turn in enumerate(result.get("turns", []), 1):
+                turn_status = turn.get("status", STATUS_UNKNOWN)
+                turn_color = GREEN if turn_status == STATUS_PASS else RED if turn_status in (STATUS_FAIL, STATUS_ERROR) else YELLOW
+                print(f"{BOLD}{turn_color}Turn {t_idx}:{RESET} [{turn_status}] {turn.get('prompt', '')}")
+                _print_evaluated_item(
+                    response=turn.get("response", ""),
+                    expected_response=turn.get("expected_response", ""),
+                    evaluators_ran=turn.get("evaluators_ran", []),
+                    item_results=turn.get("results", {}),
+                    error=turn.get("error"),
+                )
+            print()
+            print(f"{BOLD}{MAGENTA}Thread {i} Summary:{RESET}")
+            print(f"  Status: {status_color}{status.upper()}{RESET}")
+            print(f"  Turns passed: {status_color}{summary.get('turns_passed', 0)}/{summary.get('turns_total', 0)}{RESET}")
+            print(f"{BLUE}{'-' * 30}{RESET}")
+        else:
+            print(f"{BOLD}{GREEN}Prompt {i}:{RESET} {result['prompt']}")
+            _print_evaluated_item(
+                response=result.get('response', ''),
+                expected_response=result.get('expected_response', ''),
+                evaluators_ran=result.get('evaluators_ran', []),
+                item_results=result.get('results', {}),
+                error=result.get('errorDetails'),
+            )
+            print(f"{BLUE}{'-' * 30}{RESET}")
 
 def extract_eval_score(data: dict, metric_id: str) -> Optional[Dict]:
     """Extract an EvalScore object from a decorated metric dict.
@@ -338,16 +955,14 @@ def extract_eval_score(data: dict, metric_id: str) -> Optional[Dict]:
     DEFAULT_THRESHOLD = 3  # fallback; decorate_metric should always set this
 
     score_val = None
-    for k in (metric_id, f"{metric_id}_score", "score", "value"):
-        if k in data and isinstance(data[k], (int, float)):
-            score_val = data[k]
-            break
+    if metric_id in data and isinstance(data[metric_id], (int, float)):
+        score_val = data[metric_id]
     if score_val is None:
         return None
 
     result = data.get("result")
-    if result not in ("pass", "fail"):
-        result = "pass" if score_val >= data.get("threshold", DEFAULT_THRESHOLD) else "fail"
+    if result not in (STATUS_PASS, STATUS_FAIL):
+        result = STATUS_PASS if score_val >= data.get("threshold", DEFAULT_THRESHOLD) else STATUS_FAIL
 
     eval_score: Dict[str, Any] = {
         "score": score_val,
@@ -360,48 +975,33 @@ def extract_eval_score(data: dict, metric_id: str) -> Optional[Dict]:
     return eval_score
 
 
-def convert_result_to_eval_item(result: Dict) -> Dict:
-    """Convert an internal evaluation result dict to a schema-compliant EvalItem.
+def _convert_scores_to_schema(results_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """Convert raw evaluator results to schema-compliant score objects.
 
-    Internal format (from run_evaluations):
-        {prompt, response, expected_response, results: {relevance_score: "JSON", ...}}
-    Schema EvalItem format:
-        {prompt, response, expected_response, scores: {relevance: EvalScore, ...}}
+    Evaluator results in results_dict are dicts (from _decorate_metric) or
+    None when skipped/crashed. None values are omitted from output.
     """
-    item: Dict[str, Any] = {
-        "prompt": result["prompt"],
-        "response": result["response"],
-        "expected_response": result["expected_response"],
-    }
-
     scores: Dict[str, Any] = {}
-    results_dict = result.get("results", {})
-
-    # EvalScore metrics (all share the same schema shape: {score, result, threshold})
-    # Tuple: (internal results key, metric ID for score lookup, schema output key)
-    for internal_key, metric_id, schema_key in [
-        ("relevance_score", "relevance", "relevance"),
-        ("coherence_score", "coherence", "coherence"),
-        ("groundedness_score", "groundedness", "groundedness"),
-        ("tool_call_accuracy_score", "tool_call_accuracy", "toolCallAccuracy"),
+
+    for eval_key, schema_key in [
+        (RELEVANCE, "relevance"),
+        (COHERENCE, "coherence"),
+        (GROUNDEDNESS, "groundedness"),
+        (TOOL_CALL_ACCURACY, "toolCallAccuracy"),
     ]:
-        raw = results_dict.get(internal_key)
-        if not raw:
+        data = results_dict.get(eval_key)
+        if data is None:
             continue
-        data = json.loads(raw) if isinstance(raw, str) else raw
-        eval_score = extract_eval_score(data, metric_id)
+        eval_score = extract_eval_score(data, METRIC_IDS[eval_key])
         if eval_score:
             scores[schema_key] = eval_score
 
-    # Citations → CitationScore (different schema shape: {count, result, threshold} + format)
-    raw_citations = results_dict.get("citations_score")
-    if raw_citations:
-        data = json.loads(raw_citations) if isinstance(raw_citations, str) else raw_citations
-        count = data.get("score", 0)
+    data = results_dict.get(CITATIONS)
+    if data is not None:
+        count = data.get("citations", 0)
         cit_result = data.get("result")
-        if cit_result not in ("pass", "fail"):
-            cit_result = "pass" if count >= data.get("threshold", 1) else "fail"
-
+        if cit_result not in (STATUS_PASS, STATUS_FAIL):
+            cit_result = STATUS_PASS if count >= data.get("threshold", 1) else STATUS_FAIL
         citation_score: Dict[str, Any] = {
             "count": count,
             "result": cit_result,
@@ -411,17 +1011,100 @@ def convert_result_to_eval_item(result: Dict) -> Dict:
             citation_score["format"] = data["citation_format"]
         scores["citations"] = citation_score
 
+    data = results_dict.get(EXACT_MATCH)
+    if data is not None:
+        is_match = data.get("exact_match", 0.0) == 1.0
+        scores["exactMatch"] = {
+            "match": is_match,
+            "result": data.get("result", STATUS_PASS if is_match else STATUS_FAIL),
+            "reason": data.get("exact_match_reason", ""),
+        }
+
+    data = results_dict.get(PARTIAL_MATCH)
+    if data is not None:
+        scores["partialMatch"] = {
+            "score": data.get("partial_match", 0.0),
+            "result": data.get("result", STATUS_FAIL),
+            "threshold": data.get("threshold", 0.5),
+            "reason": data.get("partial_match_reason", ""),
+        }
+
+    return scores
+
+
+def convert_result_to_eval_item(result: Dict) -> Dict:
+    """Convert an internal evaluation result dict to a schema-compliant EvalItem."""
+    item: Dict[str, Any] = {
+        "prompt": result["prompt"],
+        "response": result["response"],
+        "expected_response": result["expected_response"],
+    }
+
+    if "evaluators" in result:
+        item["evaluators"] = result["evaluators"]
+    if "evaluators_mode" in result:
+        item["evaluators_mode"] = result["evaluators_mode"]
+
+    scores = _convert_scores_to_schema(result.get("results", {}))
     if scores:
         item["scores"] = scores
 
     return item
 
 
-def write_results_to_json(results: List[Dict], output_file: str, agent_id: Optional[str] = None):
+def convert_thread_result_to_output(thread_result: Dict) -> Dict:
+    """Convert a multi-turn thread result to the output format."""
+    output_turns = []
+    for turn in thread_result.get("turns", []):
+        output_turn: Dict[str, Any] = {"prompt": turn.get("prompt", "")}
+        if "expected_response" in turn:
+            output_turn["expected_response"] = turn["expected_response"]
+        if "response" in turn:
+            output_turn["response"] = turn["response"]
+        if "status" in turn:
+            output_turn["status"] = turn["status"]
+        if "error" in turn:
+            output_turn["error"] = turn["error"]
+        if "evaluators" in turn:
+            output_turn["evaluators"] = turn["evaluators"]
+        if "evaluators_mode" in turn:
+            output_turn["evaluators_mode"] = turn["evaluators_mode"]
+
+        scores = _convert_scores_to_schema(turn.get("results", {}))
+        if scores:
+            output_turn["scores"] = scores
+
+        output_turns.append(output_turn)
+
+    output: Dict[str, Any] = {}
+    if thread_result.get("name"):
+        output["name"] = thread_result["name"]
+    if thread_result.get("description"):
+        output["description"] = thread_result["description"]
+    if thread_result.get("conversation_id"):
+        output["conversation_id"] = thread_result["conversation_id"]
+    output["turns"] = output_turns
+    if thread_result.get("summary"):
+        output["summary"] = thread_result["summary"]
+
+    return output
+
+
+def convert_result_to_output_item(result: Dict) -> Dict:
+    """Convert an internal result dict to an output item. Routes by type."""
+    if result.get("type") == "multi_turn":
+        return convert_thread_result_to_output(result)
+    return convert_result_to_eval_item(result)
+
+
+def write_results_to_json(results: List[Dict], output_file: str, agent_id: Optional[str] = None,
+                          default_evaluators: Optional[Dict[str, Any]] = None,
+                          agent_name: Optional[str] = None,
+                          cli_version: Optional[str] = None):
     """Write results to a schema-compliant eval document JSON file.
 
     Output follows the eval-document.schema.json format:
-    {schemaVersion, metadata, items: [EvalItem]}
+    {schemaVersion, metadata, default_evaluators?, items: [EvalItem]}
     """
     try:
         try:
@@ -429,52 +1112,145 @@ def write_results_to_json(results: List[Dict], output_file: str, agent_id: Optio
         except Exception:
             current_version = "1.0.0"
 
-        items = [convert_result_to_eval_item(r) for r in results]
+        items = [convert_result_to_output_item(r) for r in results]
 
         metadata: Dict[str, Any] = {
             "evaluatedAt": datetime.now(timezone.utc).isoformat(),
         }
         if agent_id:
             metadata["agentId"] = agent_id
+        if agent_name:
+            metadata["agentName"] = agent_name
+        if cli_version:
+            metadata["cliVersion"] = cli_version
 
         output_data: Dict[str, Any] = {
             "schemaVersion": current_version,
             "metadata": metadata,
-            "items": items,
         }
 
+        if default_evaluators is not None:
+            output_data["default_evaluators"] = default_evaluators
+
+        output_data["items"] = items
+
         with open(output_file, 'w', encoding='utf-8') as f:
             json.dump(output_data, f, indent=2, ensure_ascii=False)
-        print(f"Results saved to {output_file}")
+        emit_structured_log("info", f"Results saved to {output_file}", operation=Operation.WRITE_OUTPUT)
     except Exception as e:
-        print(f"Error writing to JSON file: {e}")
+        emit_structured_log("error", f"Error writing to JSON file: {e}", operation=Operation.WRITE_OUTPUT)
         sys.exit(1)
 
-def write_results_to_csv(results: List[Dict], output_file: str):
+def _results_to_csv_json(results_dict: Dict) -> str:
+    """Serialize evaluator results dict to a CSV-safe JSON string.
+
+    Skips None (crashed/skipped evaluators). Results are dicts produced
+    by _decorate_metric.
+    """
+    if not results_dict:
+        return ""
+    non_null = {k: v for k, v in results_dict.items() if v is not None}
+    return json.dumps(non_null) if non_null else ""
+
+
+def write_results_to_csv(results: List[Dict], output_file: str,
+                         agent_name: Optional[str] = None, agent_id: Optional[str] = None,
+                         cli_version: Optional[str] = None):
     """Write results to CSV file."""
     try:
         with open(output_file, 'w', newline='', encoding='utf-8') as f:
             if results:
-                # Write aggregate statistics first if multiple results
-                if len(results) > 1:
-                    aggregates = calculate_aggregate_statistics(results)
-                    if aggregates:
+                metadata_parts = []
+                if agent_name:
+                    metadata_parts.append(f"Agent Name: {agent_name}")
+                if agent_id:
+                    metadata_parts.append(f"Agent ID: {agent_id}")
+                if cli_version:
+                    metadata_parts.append(f"CLI Version: {cli_version}")
+                if metadata_parts:
+                    f.write(f"# {' | '.join(metadata_parts)}\n")
+
+                aggregates = calculate_aggregate_statistics(results)
+                if aggregates:
+                    total_items = aggregates[next(iter(aggregates))].get('total_prompts', len(results))
+                    if total_items > 1:
                         f.write("# AGGREGATE STATISTICS\n")
-                        f.write("Metric,Pass Rate (%),Passed,Failed,Avg Score,Threshold\n")
+                        f.write("Metric,Prompts Evaluated,Total Prompts,Pass Rate (%),Passed,Failed,Avg Score,Threshold\n")
                         for metric_name, stats in aggregates.items():
                             threshold_str = str(stats.get('threshold', 'N/A'))
-                            f.write(f"{metric_name},{stats['pass_rate']:.1f},{stats['pass_count']},{stats['fail_count']},{stats['avg_score']:.2f},{threshold_str}\n")
+                            prompts_evaluated = stats.get('prompts_evaluated', stats['total_evaluated'])
+                            total_prompts = stats.get('total_prompts', total_items)
+                            f.write(f"{metric_name},{prompts_evaluated},{total_prompts},{stats['pass_rate']:.1f},{stats['pass_count']},{stats['fail_count']},{stats['avg_score']:.2f},{threshold_str}\n")
                         f.write("\n# INDIVIDUAL RESULTS\n")
-                
-                # Write individual results
-                writer = csv.DictWriter(f, fieldnames=results[0].keys())
-                writer.writeheader()
-                writer.writerows(results)
-        print(f"Results saved to {output_file}")
+
+                single_turn_rows = []
+                multi_turn_rows = []
+                for result in results:
+                    if result.get("type") == "multi_turn":
+                        thread_name = result.get("name", "")
+                        for turn_idx, turn in enumerate(result.get("turns", [])):
+                            multi_turn_rows.append({
+                                "thread_name": thread_name,
+                                "turn_index": turn_idx + 1,
+                                "prompt": turn.get("prompt", ""),
+                                "response": turn.get("response", ""),
+                                "expected_response": turn.get("expected_response", ""),
+                                "status": turn.get("status", ""),
+                                "error": turn.get("error", ""),
+                                "scores": _results_to_csv_json(turn.get("results", {})),
+                            })
+                        summary = result.get("summary", {})
+                        multi_turn_rows.append({
+                            "thread_name": thread_name,
+                            "turn_index": "summary",
+                            "prompt": "",
+                            "response": "",
+                            "expected_response": "",
+                            "status": summary.get("overall_status", ""),
+                            "scores": f"{summary.get('turns_passed', 0)}/{summary.get('turns_total', 0)} turns passed",
+                        })
+                    else:
+                        exclude_keys = {'evaluators_ran', 'evaluators', 'evaluators_mode', '_enhanced_response', 'results'}
+                        row = {k: v for k, v in result.items() if k not in exclude_keys}
+                        if "results" in result:
+                            row["scores"] = _results_to_csv_json(result["results"])
+                        single_turn_rows.append(row)
+
+                if single_turn_rows:
+                    if multi_turn_rows:
+                        f.write("# SINGLE-TURN RESULTS\n")
+                    fieldnames = list(single_turn_rows[0].keys())
+                    for row in single_turn_rows:
+                        for k in row:
+                            if k not in fieldnames:
+                                fieldnames.append(k)
+                    writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
+                    writer.writeheader()
+                    writer.writerows(single_turn_rows)
+
+                if multi_turn_rows:
+                    if single_turn_rows:
+                        f.write("\n")
+                    f.write("# MULTI-TURN RESULTS\n")
+                    fieldnames = ["thread_name", "turn_index", "prompt", "response", "expected_response", "status", "error", "scores"]
+                    writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
+                    writer.writeheader()
+                    writer.writerows(multi_turn_rows)
+            emit_structured_log("info", f"Results saved to {output_file}", operation=Operation.WRITE_OUTPUT)
     except Exception as e:
-        print(f"Error writing to CSV file: {e}")
+        emit_structured_log("error", f"Error writing to CSV file: {e}", operation=Operation.WRITE_OUTPUT)
         sys.exit(1)
 
+def normalize_agent_id(agent_id):
+    """Append '.declarativeAgent' if agent_id has no '.', else return unchanged.
+
+    Returns the input unchanged when it is None/empty or already contains a dot.
+    """
+    if not agent_id:
+        return agent_id
+    return agent_id if '.' in agent_id else f"{agent_id}.declarativeAgent"
+
+
 def parse_arguments():
     """Parse command line arguments."""
     parser = argparse.ArgumentParser(
@@ -503,8 +1279,8 @@ Examples:
   # Save results to HTML and open in browser
   python main.py --output report.html
 
-  # Verbose output
-  python main.py --verbose
+  # Debug-level diagnostics
+  python main.py --log-level debug
 
   # Sign out and clear cached authentication tokens
   python main.py --signout
@@ -553,28 +1329,41 @@ Examples:
     
     # Behavior options
     parser.add_argument(
-        '--verbose', 
-        action='store_true', 
-        help='Enable verbose output'
-    )
-    parser.add_argument(
-        '--quiet', 
-        action='store_true', 
-        help='Suppress non-essential output'
-    )
-    parser.add_argument(
-        '--citation-format',
-        choices=['oai_unicode', 'legacy_bracket'],
-        default='oai_unicode',
-        help='Citation format to detect. "oai_unicode" for new OAI format (default), "legacy_bracket" for old [^i^] format'
+        '--log-level',
+        nargs='?',
+        const='info',
+        action='append',
+        help='Set log verbosity: debug, info, warning, error. Bare --log-level resolves to info.'
     )
+
     parser.add_argument(
         '--signout',
         action='store_true',
         help='Sign out and clear cached authentication tokens'
     )
+
+    parser.add_argument(
+        '--concurrency',
+        type=int,
+        default=5,
+        help='Number of parallel workers for prompt processing (1-5, default: 5)'
+    )
     
-    return parser.parse_args()
+    args = parser.parse_args()
+
+    args.m365_agent_id = normalize_agent_id(args.m365_agent_id)
+
+    if args.concurrency < 1:
+        parser.error('--concurrency must be an integer >= 1.')
+    if args.concurrency > MAX_CONCURRENCY:
+        emit_structured_log(
+            "warning",
+            f"--concurrency {args.concurrency} exceeds max {MAX_CONCURRENCY}; clamping to {MAX_CONCURRENCY}.",
+            operation=Operation.SETUP,
+        )
+        args.concurrency = MAX_CONCURRENCY
+
+    return args
 
 def validate_environment() -> CallPath:
     """Validate required environment variables."""
@@ -583,25 +1372,40 @@ def validate_environment() -> CallPath:
         "AZURE_AI_API_KEY",
         "AZURE_AI_API_VERSION",
         "AZURE_AI_MODEL_NAME",
-        # Chat API specific
-        "COPILOT_API_ENDPOINT",
-        "X_SCENARIO_HEADER"
     ]
 
     if os.environ.get("COPILOT_API_ACCESS_TOKEN"):
         call_path = CallPath.ACCESS_TOKEN
-        required_env_vars.append("COPILOT_API_ACCESS_TOKEN")
+        required_env_vars.extend([
+            "COPILOT_API_ACCESS_TOKEN",
+            "COPILOT_API_ENDPOINT",
+            "X_SCENARIO_HEADER",
+        ])
+    elif os.environ.get("WORK_IQ_A2A_ENDPOINT"):
+        call_path = CallPath.A2A
+        required_env_vars.extend([
+            "WORK_IQ_A2A_ENDPOINT",
+            "WORK_IQ_A2A_CLIENT_ID",
+            "TENANT_ID",
+        ])
     else:
         call_path = CallPath.COPILOT_AUTH
         required_env_vars.extend([
+            "COPILOT_API_ENDPOINT",
+            "X_SCENARIO_HEADER",
             "M365_EVAL_CLIENT_ID",
-            "TENANT_ID"
+            "TENANT_ID",
         ])
     
     missing_vars = [var for var in required_env_vars if not os.environ.get(var)]
     if missing_vars:
-        print(f"Error: Missing required environment variables: {', '.join(missing_vars)}")
-        print("Please ensure your .env file contains all required Azure configuration.")
+        emit_structured_log(
+            "error",
+            "Missing required environment variables: "
+            f"{', '.join(missing_vars)}. Please ensure your .env file contains "
+            "all required Azure configuration.",
+            operation=Operation.VALIDATE_ENV,
+        )
         sys.exit(1)
     return call_path
 
@@ -635,78 +1439,58 @@ def validate_endpoint_url(url: str, allowed_domains: List[str]) -> bool:
         # Convert other parsing errors to ValueError
         raise ValueError(f"Invalid URL format: {url}") from e
 
-def get_prompt_datasets(args) -> Tuple[List[str], List[str]]:
-    """Get prompts and expected responses based on command line arguments."""
+def get_prompt_datasets(args) -> Tuple[List[Dict], Optional[Dict]]:
+    """Get prompts and expected responses based on command line arguments.
+
+    Returns:
+        Tuple of (eval_items, default_evaluators).
+    """
     if args.prompts:
         if args.expected and len(args.prompts) != len(args.expected):
-            print("Error: Number of prompts must match number of expected responses")
+            emit_structured_log(
+                "error",
+                "Number of prompts must match number of expected responses. "
+                "Update --expected values to match the prompt count.",
+            )
             sys.exit(1)
-        prompts = args.prompts
-        expected_responses = args.expected or [""] * len(prompts)
+        expected_responses = args.expected or [""] * len(args.prompts)
+        eval_items = [
+            {"prompt": p, "expected_response": e}
+            for p, e in zip(args.prompts, expected_responses)
+        ]
+        return eval_items, None
     elif args.prompts_file:
-        prompts, expected_responses = load_prompts_from_file(args.prompts_file)
+        return load_prompts_from_file(args.prompts_file)
     elif args.interactive:
         prompts, expected_responses = get_interactive_prompts()
+        eval_items = [
+            {"prompt": p, "expected_response": e}
+            for p, e in zip(prompts, expected_responses)
+        ]
+        return eval_items, None
     else:
-        # Use default prompts
         prompts, expected_responses = get_default_prompts_and_responses()
-    
-    return prompts, expected_responses
+        eval_items = [
+            {"prompt": p, "expected_response": e}
+            for p, e in zip(prompts, expected_responses)
+        ]
+        return eval_items, None
 
-def fetch_available_agents(copilot_api_endpoint: str, access_token: str, user_oid: str) -> List[Dict[str, Any]]:
-    """
-    Fetch available agents for the user from the Copilot API.
-    
-    Args:
-        access_token: Bearer token for API authentication
-        user_oid: User object ID for agent filtering
-        
-    Returns:
-        List of agent dictionaries.
-    """
-    request_headers = {
-        "Content-Type": "application/json",
-        "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
-        "Authorization": f"Bearer {access_token}"
-    }
-    
-    try:
-        # Build the query parameter with participant info
-        request_data = json.dumps({"participant": {"id": user_oid}})
-        query_param = urllib.parse.quote(request_data)
-        
-        # Try to fetch agents from /GetGptList endpoint
-        req = urllib.request.Request(
-            f"{copilot_api_endpoint}/GetGptList?request={query_param}",
-            headers=request_headers,
-            method="GET"
-        )
-        with urllib.request.urlopen(req, timeout=120) as resp:
-            data = json.loads(resp.read().decode("utf-8"))
-            agents = data.get("gptList", [])
-            return agents
-    except urllib.error.HTTPError as e:
-        # If endpoint doesn't exist or returns error, return empty list
-        print(f"Warning: Unable to fetch agents list (HTTP {e.code}).")
-        return []
-    except Exception as e:
-        print(f"Warning: Error fetching agents: {e}")
-        return []
-
-def select_agent_interactively(agents: List[Dict[str, Any]]) -> Optional[str]:
+def select_agent_interactively(agents: List[Dict[str, Any]]) -> Tuple[Optional[str], Optional[str]]:
     """
     Display an interactive agent selector using questionary.
-    
+
     Args:
         agents: List of agent dictionaries.
-        
+
     Returns:
-        Selected agent ID or None if cancelled/skipped
+        Tuple of (agent_id, agent_name) or (None, None) if cancelled/skipped
     """
     if not agents:
-        return None
-    
-    # Create choices for questionary
+        return None, None
+
+    # Build id→name lookup and choices
+    id_to_name: Dict[str, str] = {}
     choices = []
     sorted_agents = sorted(agents, key=lambda x: (not x.get('isOwner', False), x.get('name', '')))
     for agent in sorted_agents:
@@ -714,12 +1498,13 @@ def select_agent_interactively(agents: List[Dict[str, Any]]) -> Optional[str]:
         agent_name = agent.get("name", "Unknown")
         agent_description = agent.get("description", "Unknown")
         agent_is_owner = agent.get('isOwner')
-        
+        id_to_name[agent_id] = agent_name
+
         # Format the display text
         display_text = f"{agent_name} ({agent_id}, IsOwner: {agent_is_owner}) - {agent_description}"
-        
+
         choices.append(questionary.Choice(title=display_text, value=agent_id))
-    
+
     # Display the selection prompt
     selected_agent = questionary.select(
         "Select an agent to evaluate:",
@@ -727,238 +1512,260 @@ def select_agent_interactively(agents: List[Dict[str, Any]]) -> Optional[str]:
         use_shortcuts=True,
         use_arrow_keys=True
     ).ask()
-    
-    return selected_agent
-
-@functools.lru_cache(maxsize=1)
-def _get_iana_timezone_name() -> str:
-    """Get the IANA timezone name from the system using tzlocal.
-
-    Tries get_localzone_name() first; falls back to str(get_localzone()) when the
-    former raises (e.g. no zone configured on some Unix systems).  Result is cached
-    after the first call so tzlocal is only invoked once per session.
-    """
-    try:
-        return tzlocal.get_localzone_name()
-    except Exception:
-        return str(tzlocal.get_localzone())
-
-
-@functools.lru_cache(maxsize=1)
-def _get_location_info() -> Dict[str, Any]:
-    """Return a locationInfo dict containing the local UTC offset and IANA timezone name.
 
-    Result is cached after the first call so the computation runs only once per session.
-    """
-    now = datetime.now().astimezone()
-    utc_offset = now.utcoffset()
-    offset_hours = int(utc_offset.total_seconds() // 3600) if utc_offset is not None else 0
-    return {
-        "timeZoneOffset": offset_hours,
-        "timeZone": _get_iana_timezone_name(),
-    }
+    return selected_agent, id_to_name.get(selected_agent) if selected_agent else None
 
 
-def build_chat_payload(prompt: str, user_oid: str, agent_id: str|None) -> bytes:
-    message = {
-        "message": {
-            "text": prompt,
-            "author": "user",
-            "messageType": "chat",
-            "timestamp": datetime.now(timezone.utc).isoformat(),
-            "locationInfo": _get_location_info(),
-            "from": {
-                    "id": user_oid,
-            }
-        },
-        "verbosity": "verbose", # To enable detailed telemetry in response (to extract tool usage, etc.)
-    }
-    
-    if agent_id:
-            message["gpts"] = [
-                {
-                        "id": agent_id.strip(),
-                        "source": "MOS3"
-                }
-            ]
-            message["optionsSets"] = [
-                "disable_action_confirmation"   # Disable 3P action confirmation prompts for agents while scraping
-            ]
-  
-    return json.dumps(message).encode("utf-8")
-
-def send_prompt_to_agent_in_sydney(prompts: List[str], copilot_api_endpoint: str, access_token: str, user_oid: str, args) -> Dict[str, Dict[str, any]]:
-    """ Send prompts to the chat API and return enhanced responses. """
-
-    request_headers = {
-        "Content-Type": "application/json",
-        "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
-        "Authorization": f"Bearer {access_token}"
-        }
-    raw_responses: Dict[str, str] = {}
-    for i, prompt in enumerate(prompts, 1):
-        if not args.quiet:
-            print(f"Processing prompt {i}/{len(prompts)}...")
-
-        # Build the payload
-        payload = build_chat_payload(prompt, user_oid, args.m365_agent_id)
-        if args.verbose:
-            print(f"[Sydney] Sending payload: {payload.decode('utf-8')}")
-
-        # Send the request to /chat
-        req = urllib.request.Request(f"{copilot_api_endpoint}/chat", data=payload, headers=request_headers, method="POST")
-        try:
-            with urllib.request.urlopen(req, timeout=120) as resp:
-                raw = resp.read().decode("utf-8", errors="replace")
-        except urllib.error.HTTPError as e:
-            error_body = None
-            try:
-                error_body = e.read().decode("utf-8", errors="replace")
-            except Exception:
-                pass
-            msg = f"Chat API request failed (HTTP {e.code} {e.reason})."
-            if error_body:
-                msg += f" Body: {error_body[:500]}"
-            raise RuntimeError(msg) from e
-        except urllib.error.URLError as e:
-            raise RuntimeError(f"Chat API connection error: {getattr(e, 'reason', str(e))}") from e
-        
-        if args.verbose:
-            print(f"[Sydney] Raw response: {raw}")
-        
-        # Store raw response for enhancement
-        raw_responses[prompt] = raw.strip()
-    
-    # Extract enhanced responses using the new extractor
-    enhanced_responses = extract_enhanced_responses(raw_responses)
-    return enhanced_responses
-
-def output_results(results: List[Dict], args):
+def output_results(results: List[Dict], args, default_evaluators: Optional[Dict[str, Any]] = None,
+                    agent_name: Optional[str] = None, cli_version: Optional[str] = None):
     """Output results based on specified format."""
+    metadata_kwargs = dict(
+        agent_name=agent_name,
+        agent_id=getattr(args, 'm365_agent_id', None),
+        cli_version=cli_version,
+    )
     if args.output:
         output_lower = args.output.lower()
         if output_lower.endswith('.json'):
-            write_results_to_json(results, args.output, agent_id=getattr(args, 'm365_agent_id', None))
+            write_results_to_json(results, args.output, default_evaluators=default_evaluators,
+                                  **metadata_kwargs)
         elif output_lower.endswith('.csv'):
-            write_results_to_csv(results, args.output)
+            write_results_to_csv(results, args.output, **metadata_kwargs)
         elif output_lower.endswith('.html'):
-            write_results_to_html(results, args.output)
+            write_results_to_html(results, args.output, **metadata_kwargs)
             abs_path = os.path.abspath(args.output)
             webbrowser.open(f'file://{abs_path}')
         else:
-            write_results_to_json(results, args.output, agent_id=getattr(args, 'm365_agent_id', None))
+            write_results_to_json(results, args.output, default_evaluators=default_evaluators,
+                                  **metadata_kwargs)
     else:
-        write_results_to_console(results)
+        write_results_to_console(results, **metadata_kwargs)
 
 def main():
     """Main function to orchestrate the evaluation process."""
     load_dotenv()
     args = parse_arguments()
 
+    effective_log_level, error_message = resolve_log_level(args.log_level)
+    if error_message:
+        print(error_message)
+        print(
+            "Next step: rerun with --log-level {debug|info|warning|error}. "
+            "For support, share the console diagnostics output from this run."
+        )
+        sys.exit(2)
+
+    args.effective_log_level = effective_log_level
+    configure_cli_logging(effective_log_level)
+    emit_structured_log("info", f"Log level set to '{effective_log_level}'.", operation=Operation.SETUP)
+
     # Check minimum version before proceeding
-    cli_version = get_cli_version(quiet=args.quiet)
-    if not should_bypass_min_version_check(args) and not check_min_version(cli_version, quiet=args.quiet):
+    quiet_for_version = effective_log_level in ("warning", "error")
+    cli_version = get_cli_version(quiet=quiet_for_version)
+    if not should_bypass_min_version_check(args) and not check_min_version(cli_version, quiet=quiet_for_version):
         sys.exit(1)
 
     # Validate environment variables required for evaluation
     call_path = validate_environment()
-    copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
-    validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
     
     user_oid = ""
 
-    if call_path == CallPath.ACCESS_TOKEN:
-        access_token = os.environ["COPILOT_API_ACCESS_TOKEN"]
-    else:
-      scopes_str = os.environ.get(
-          "COPILOT_SCOPES", "https://substrate.office.com/sydney/.default"
-          )
-          
-      auth_handler = AuthHandler(
-          client_id=os.environ["M365_EVAL_CLIENT_ID"],
-          tenant_id=os.environ["TENANT_ID"],
-          scopes_str=scopes_str
-          )
-
-      # Signout user
-      if args.signout:
-          try:
-              auth_handler.clear_cache()
-          except Exception as e:
-              print(f"Error during signout: {e}")
-              sys.exit(1)
-          sys.exit(0)
-
-      # Authenticate before loading prompts
-      try:
-        auth_result = auth_handler.acquire_token_interactive() or {}
-        access_token = auth_result.get("access_token") or ""
-        if not access_token:
-            raise RuntimeError("Failed to acquire access token from authentication result")
-
-        id_token_claims = auth_result.get("id_token_claims")
-        if not isinstance(id_token_claims, dict):
-            print("id_token_claims is missing or invalid in authentication result")
-        else:
-            user_oid = id_token_claims.get("oid") or ""
-        
-      except Exception as e:
-        print(f"\033[91mError during authentication: {e}\033[0m")
-        if args.verbose:
-            import traceback
-            traceback.print_exc()
-        sys.exit(1)
+    match call_path:
+        case CallPath.ACCESS_TOKEN:
+            access_token = os.environ["COPILOT_API_ACCESS_TOKEN"]
+            user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
+            copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
+            validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
+            agent_client = SydneyClient(
+                copilot_api_endpoint=copilot_api_endpoint,
+                access_token=access_token,
+                user_oid=user_oid,
+                logger=CLI_LOGGER,
+                diagnostic_records=DIAGNOSTIC_RECORDS,
+            )
 
-    if not user_oid and access_token:
-        # Fallback: extract from access token.
-        user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
+        case CallPath.A2A:
+            emit_structured_log(
+                "warning",
+                "The A2A endpoint is experimental and may change without notice.",
+                operation=Operation.SETUP,
+            )
+            a2a_endpoint = os.environ["WORK_IQ_A2A_ENDPOINT"]
+            validate_endpoint_url(a2a_endpoint, ALLOWED_ENDPOINTS)
+
+            a2a_scopes_str = os.environ.get("WORK_IQ_A2A_SCOPES", "")
+            a2a_auth_handler = AuthHandler(
+                client_id=os.environ["WORK_IQ_A2A_CLIENT_ID"],
+                tenant_id=os.environ["TENANT_ID"],
+                scopes_str=a2a_scopes_str,
+            )
+            try:
+                a2a_auth_result = a2a_auth_handler.acquire_token_interactive() or {}
+                a2a_access_token = a2a_auth_result.get("access_token") or ""
+                if not a2a_access_token:
+                    raise RuntimeError("Failed to acquire A2A access token")
+            except Exception as e:
+                emit_structured_log(
+                    "error",
+                    f"Error during A2A authentication: {e}",
+                    operation=Operation.AUTHENTICATE,
+                )
+                if effective_log_level == "debug":
+                    import traceback
+                    traceback.print_exc()
+                sys.exit(1)
+            try:
+                agent_client = A2AClient(
+                    a2a_endpoint=a2a_endpoint,
+                    access_token=a2a_access_token,
+                    logger=CLI_LOGGER,
+                    diagnostic_records=DIAGNOSTIC_RECORDS,
+                )
+            except Exception as e:
+                emit_structured_log(
+                    "error",
+                    f"Failed to initialize A2A client: {e}",
+                    operation=Operation.SETUP,
+                )
+                sys.exit(1)
+
+        case CallPath.COPILOT_AUTH:
+            copilot_api_endpoint = os.environ["COPILOT_API_ENDPOINT"]
+            validate_endpoint_url(copilot_api_endpoint, ALLOWED_ENDPOINTS)
+            auth_handler = AuthHandler(
+                client_id=os.environ["M365_EVAL_CLIENT_ID"],
+                tenant_id=os.environ["TENANT_ID"],
+                scopes_str=os.environ.get("COPILOT_SCOPES", ""),
+            )
+
+            # Signout user
+            if args.signout:
+                try:
+                    auth_handler.clear_cache()
+                except Exception as e:
+                    emit_structured_log("error", f"Error during signout: {e}", operation=Operation.AUTHENTICATE)
+                    sys.exit(1)
+                sys.exit(0)
 
-    # 1. Load evaluation datasets (prompts and expected_responses)
-    prompts, expected_responses = get_prompt_datasets(args)
+            try:
+                auth_result = auth_handler.acquire_token_interactive() or {}
+                access_token = auth_result.get("access_token") or ""
+                if not access_token:
+                    raise RuntimeError("Failed to acquire access token from authentication result")
+
+                id_token_claims = auth_result.get("id_token_claims")
+                if not isinstance(id_token_claims, dict):
+                    emit_structured_log(
+                        "warning", "id_token_claims is missing or invalid in authentication result",
+                        operation=Operation.AUTHENTICATE,
+                    )
+                else:
+                    user_oid = id_token_claims.get("oid") or ""
+
+                if not user_oid:
+                    user_oid = AuthHandler.extract_user_oid_from_access_token(access_token)
 
-    if not args.quiet:
-        print(f"Running evaluation on {len(prompts)} prompt(s)...")
+            except Exception as e:
+                emit_structured_log("error", f"Error during authentication: {e}", operation=Operation.AUTHENTICATE)
+                if effective_log_level == "debug":
+                    import traceback
+                    traceback.print_exc()
+                sys.exit(1)
+
+            agent_client = SydneyClient(
+                copilot_api_endpoint=copilot_api_endpoint,
+                access_token=access_token,
+                user_oid=user_oid,
+                logger=CLI_LOGGER,
+                diagnostic_records=DIAGNOSTIC_RECORDS,
+            )
 
+    # 1. Load evaluation datasets
+    eval_items, file_default_evaluators = get_prompt_datasets(args)
+    default_evaluators = resolve_default_evaluators(file_default_evaluators)
+
+    if effective_log_level in ("info", "debug"):
+        multi_turn_count = sum(1 for item in eval_items if "turns" in item)
+        single_turn_count = len(eval_items) - multi_turn_count
+        emit_structured_log(
+            "info",
+            f"Running evaluation on {len(eval_items)} item(s) "
+            f"({single_turn_count} single-turn, {multi_turn_count} multi-turn).",
+            operation=Operation.SETUP,
+        )
+
+    agent_name = None
     try:
-        # 3. Agent selection - if no agent ID provided, prompt user to select
+        # 2. Agent selection - when no agent ID is provided, discover agents
+        # via the active client (A2A or REST) and prompt interactively.
         if not args.m365_agent_id:
-            if not args.quiet:
-                print("No agent ID provided. Fetching available agents...")
-            
-            available_agents = fetch_available_agents(copilot_api_endpoint, access_token, user_oid)
-            if not available_agents:
-              print("No agents are available for interactive selection. Please re-run with --m365-agent-id or set the M365_AGENT_ID environment variable.")
-              sys.exit(1)
-            
-            if available_agents:
-                selected_agent_id = select_agent_interactively(available_agents)
-                if selected_agent_id:
-                    args.m365_agent_id = selected_agent_id
-                    if not args.quiet:
-                        print(f"Selected agent: {args.m365_agent_id}")
-                else:
-                    print("No agent selected. Please re-run with --m365-agent-id or set the M365_AGENT_ID environment variable.")
-                    sys.exit(1)
+            if effective_log_level in ("info", "debug"):
+                emit_structured_log("info", "No agent ID provided. Fetching available agents.", operation=Operation.FETCH_AGENTS)
 
-        # 4. Send prompts to chat API
-        responses = send_prompt_to_agent_in_sydney(prompts, copilot_api_endpoint, access_token, user_oid, args)
+            available_agents = agent_client.fetch_available_agents()
+            if not available_agents:
+                emit_structured_log(
+                  "error",
+                  "No agents are available for interactive selection. Re-run with "
+                  "--m365-agent-id or set M365_AGENT_ID.",
+                  operation=Operation.FETCH_AGENTS,
+                )
+                sys.exit(1)
+
+            selected_agent_id, agent_name = select_agent_interactively(available_agents)
+            if selected_agent_id:
+                args.m365_agent_id = selected_agent_id
+                if effective_log_level in ("info", "debug"):
+                    emit_structured_log("info", f"Selected agent: {args.m365_agent_id}", operation=Operation.FETCH_AGENTS)
+            else:
+                emit_structured_log(
+                        "error",
+                        "No agent selected. Re-run with --m365-agent-id or set M365_AGENT_ID.",
+                        operation=Operation.FETCH_AGENTS,
+                )
+                sys.exit(1)
     except Exception as e:
-        print(f"\033[91mError sending prompts to chat API: {e}\033[0m")
-        if args.verbose:
+        emit_structured_log("error", f"Error during agent discovery: {e}", operation=Operation.FETCH_AGENTS)
+        if effective_log_level == "debug":
             import traceback
             traceback.print_exc()
         sys.exit(1)
+
+    # Pre-resolve agent endpoint (A2A agent card lookup; no-op for REST)
+    if args.m365_agent_id:
+        agent_client.resolve_agent(args.m365_agent_id)
+
+    # 3. Build pipeline config and run evaluation pipeline
+    model_config = AzureOpenAIModelConfiguration(
+        azure_endpoint=os.environ.get("AZURE_AI_OPENAI_ENDPOINT"),
+        api_key=os.environ.get("AZURE_AI_API_KEY"),
+        api_version=os.environ.get("AZURE_AI_API_VERSION"),
+        azure_deployment=os.environ.get("AZURE_AI_MODEL_NAME"),
+    )
+    has_azure_openai = bool(
+        os.environ.get("AZURE_AI_OPENAI_ENDPOINT")
+        and os.environ.get("AZURE_AI_API_KEY")
+    )
+
+    pipeline = PipelineConfig(
+        agent_client=agent_client,
+        model_config=model_config,
+        has_azure_openai=has_azure_openai,
+        default_evaluators=default_evaluators,
+    )
+
+    results = run_pipeline(pipeline, eval_items, args)
     
-    # 5. Run evaluations
-    if not args.quiet:
-        print("Running evaluations...")
-    results = run_evaluations(args, responses, expected_responses)
-    
-    # 6. Output results
-    output_results(results, args)
+    # 4. Output results
+    output_results(results, args, default_evaluators=default_evaluators,
+                   agent_name=agent_name, cli_version=str(cli_version) if cli_version else None)
     
-    if not args.quiet:
-        print(f"\nEvaluation completed successfully! Processed {len(prompts)} prompt(s).")
+    if effective_log_level in ("info", "debug"):
+        emit_structured_log(
+            "info",
+            f"Evaluation completed successfully. Processed {len(eval_items)} item(s).",
+            operation=Operation.EVALUATE,
+        )
 
 # Call the main function when script is run directly
 if __name__ == "__main__":