@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.5.0-preview.1

package/src/clients/cli/api_clients/A2A/a2a_client.py

@@ -0,0 +1,475 @@
+from __future__ import annotations
+
+import functools
+import json
+import locale
+import logging
+import re
+import urllib.error
+import urllib.request
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+from api_clients.base_agent_client import BaseAgentClient
+from cli_logging.console_diagnostics import emit_structured_log
+from cli_logging.logging_utils import Operation
+
+# Feature flag required by the experimental A2A surface.
+_A2A_FEATURE_FLAG = "feature.EnableA2AServer"
+
+_REQUEST_TIMEOUT_SECS = 120
+
+# OAI citation marker patterns — compiled once at module level.
+# Marker format: \ue200cite(\ue202turn{X}search{Y})+\ue201
+_CITATION_REF_PAT = re.compile(r"\ue202turn\d+search(\d+)")
+_CITATION_BLOCK_PAT = re.compile(r"\ue200cite(?:\ue202turn\d+search\d+)+\ue201")
+
+
+class A2AClient(BaseAgentClient):
+    """A2A (Agent-to-Agent) JSON-RPC 2.0 client for Work IQ agents."""
+
+    def __init__(
+        self,
+        *,
+        a2a_endpoint: str,
+        access_token: str,
+        logger: Optional[logging.Logger] = None,
+        diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """
+        Args:
+            a2a_endpoint: Base URL of the A2A endpoint.
+            access_token: Bearer token for A2A authentication.
+            logger: Logger to use. Defaults to a module-level logger if not provided.
+            diagnostic_records: List to accumulate structured log entries.
+        """
+        self._endpoint = a2a_endpoint.rstrip("/")
+        self._access_token = access_token
+        self._logger = logger or logging.getLogger(__name__)
+        self._diagnostic_records = diagnostic_records
+        self._resolved_agent_url: Optional[str] = None
+
+    # ------------------------------------------------------------------ #
+    #  BaseAgentClient implementation                                     #
+    # ------------------------------------------------------------------ #
+
+    def resolve_agent(self, agent_id: str) -> None:
+        """Pre-resolve agent URL from agent card and cache for the session."""
+        self._resolved_agent_url = self._resolve_agent_url(agent_id)
+
+    def fetch_available_agents(self) -> List[Dict[str, Any]]:
+        """Fetch agents from the A2A discovery endpoint.
+
+        Calls GET {endpoint}/.agents. Each A2A agent card is normalized to
+        include 'gptId', 'name', and 'provider' so it is compatible with
+        the shared select_agent_interactively selector.
+
+        Returns an empty list if the endpoint is unreachable or returns an
+        error.
+        """
+        try:
+            agents_url = f"{self._endpoint}/.agents"
+            headers = self._build_request_headers()
+            emit_structured_log(
+                "debug",
+                f"[A2A] Fetching available agents from: {agents_url}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            req = urllib.request.Request(agents_url, headers=headers, method="GET")
+            with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                agents = json.loads(resp.read().decode("utf-8"))
+            emit_structured_log(
+                "debug",
+                f"[A2A] Available agents response: {json.dumps(agents)}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records
+            )
+            return [self._normalize_agent_card(a) for a in agents]
+        except urllib.error.HTTPError as e:
+            emit_structured_log(
+                "warning",
+                f"[A2A] Unable to fetch agents list (HTTP {e.code}).",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            return []
+        except Exception as e:
+            emit_structured_log(
+                "warning",
+                f"[A2A] Error fetching agents: {e}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            return []
+
+    @staticmethod
+    def _normalize_agent_card(agent: Dict[str, Any]) -> Dict[str, Any]:
+        """Normalize an A2A agent card to the shape expected by the selector.
+        """
+        return {
+            "gptId": agent.get("agentId"),
+            "name": agent.get("name"),
+            "provider": agent.get("provider")
+        }
+
+    def send_prompt(
+        self,
+        prompt: str,
+        agent_id: str | None = None,
+        conversation_context: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+        """Send a single prompt to the A2A endpoint and return the response with context.
+
+        Args:
+            prompt: The prompt string to send.
+            agent_id: Target agent ID. Required for A2A — no fallback discovery.
+            conversation_context: Context from a previous turn (contains context_id),
+                or None for the first turn / single-turn usage.
+
+        Returns:
+            Tuple of (enhanced_response_dict, conversation_context).
+        """
+        agent_id = (agent_id or "").strip()
+        if not agent_id:
+            raise ValueError("agent_id is required for A2A requests.")
+
+        headers = self._build_request_headers(include_content_type=True)
+        agent_url = self._resolved_agent_url or self._resolve_agent_url(agent_id)
+
+        context_id = conversation_context.get("context_id") if conversation_context else None
+
+        emit_structured_log(
+            "debug",
+            "[A2A] Sending prompt to agent.",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+
+        payload = self._build_chat_payload(prompt, context_id)
+
+        emit_structured_log(
+            "debug",
+            f"[A2A] Sending to {agent_url}: {payload.decode('utf-8')[:500]}",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+
+        result_dict, raw_result = self._send_and_parse_message(agent_url, payload, headers)
+
+        # Build updated context for subsequent turns
+        new_context_id = context_id
+        if raw_result:
+            new_context_id = raw_result.get("contextId") or context_id
+        updated_context = {"context_id": new_context_id} if new_context_id else None
+
+        return result_dict, updated_context
+
+    # ------------------------------------------------------------------ #
+    #  Private helpers                                                    #
+    # ------------------------------------------------------------------ #
+
+    def _build_request_headers(self, *, include_content_type: bool = False) -> Dict[str, str]:
+        headers: Dict[str, str] = {
+            "Authorization": f"Bearer {self._access_token}",
+            "X-variants": _A2A_FEATURE_FLAG,
+        }
+        if include_content_type:
+            headers["Content-Type"] = "application/json"
+        return headers
+
+    def _build_chat_payload(self, prompt: str, context_id: str | None = None) -> bytes:
+        message: Dict[str, Any] = {
+            "kind": "message",
+            "role": "user",
+            "parts": [{"kind": "text", "text": prompt}],
+            "messageId": str(uuid.uuid4()),
+            "metadata": {
+                "location": self._get_a2a_location(),
+            },
+        }
+        if context_id:
+            message["contextId"] = context_id
+        return json.dumps({
+            "jsonrpc": "2.0",
+            "method": "message/send",
+            "params": {"message": message},
+            "id": str(uuid.uuid4()),
+        }).encode("utf-8")
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_a2a_location() -> Dict[str, Any]:
+        locale_str = locale.getlocale()[0] or ""
+        country = locale_str.split("_")[-1] if "_" in locale_str else ""
+        return {
+            "countryOrRegion": country,
+            "countryOrRegionConfidence": 1.0,
+            "timeZone": BaseAgentClient._get_iana_timezone_name(),
+        }
+
+    def _resolve_agent_url(self, agent_id: str) -> str:
+        """Resolve the agent URL from the agent card, falling back to base URL.
+
+        """
+        headers = self._build_request_headers(include_content_type=True)
+        base_agent_url = f"{self._endpoint}/{agent_id}"
+        card_url = f"{base_agent_url}/.well-known/agent-card.json"
+        agent_url = base_agent_url
+        emit_structured_log(
+            "debug",
+            f"[A2A] Fetching agent card from: {card_url}",
+            Operation.FETCH_AGENTS,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+        try:
+            card_req = urllib.request.Request(card_url, headers=headers)
+            with urllib.request.urlopen(card_req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                raw_card = resp.read().decode("utf-8")
+            if raw_card.strip():
+                card = json.loads(raw_card)
+                agent_url = card.get("url") or base_agent_url
+        except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError, UnicodeDecodeError) as e:
+            emit_structured_log(
+                "debug",
+                f"[A2A] Agent card fetch failed ({e}); using base URL: {base_agent_url}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+        emit_structured_log(
+            "debug",
+            f"[A2A] Resolved agent URL: {agent_url}",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+        return agent_url
+
+    def _send_and_parse_message(
+        self,
+        agent_url: str,
+        payload: bytes,
+        headers: Dict[str, str],
+    ) -> tuple[Dict[str, Any], Dict[str, Any]]:
+        """Send a JSON-RPC message to the agent and parse the response.
+
+        Returns:
+            A tuple of (result_dict, raw_result) where result_dict is the
+            normalized response dict (raw_response_text, display_response_text,
+            a2a_attributions) and raw_result is the parsed JSON result object.
+
+        Raises:
+            RuntimeError: On HTTP errors, connection errors, JSON parse errors,
+                or A2A protocol errors.
+        """
+        req = urllib.request.Request(agent_url, data=payload, headers=headers, method="POST")
+        try:
+            with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                raw = resp.read().decode("utf-8", errors="replace")
+        except urllib.error.HTTPError as e:
+            body = ""
+            try:
+                body = e.read().decode("utf-8", errors="replace")
+            except Exception:
+                pass
+            raise RuntimeError(
+                f"A2A request failed (HTTP {e.code} {e.reason})."
+                + (f" Body: {body[:500]}" if body else "")
+            ) from e
+        except urllib.error.URLError as e:
+            raise RuntimeError(
+                f"A2A connection error: {getattr(e, 'reason', str(e))}"
+            ) from e
+
+        emit_structured_log(
+            "debug",
+            f"[A2A] Raw response: {raw[:500]}",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as e:
+            raise RuntimeError(f"A2A response is not valid JSON: {e}") from e
+
+        if "error" in data:
+            err = data["error"]
+            raise RuntimeError(
+                f"A2A JSON-RPC error {err.get('code', 'unknown')}: {err.get('message', 'no message')}"
+            )
+
+        if "result" not in data:
+            raise RuntimeError(
+                f"A2A response missing 'result' key. Keys present: {list(data.keys())}"
+            )
+
+        result = data["result"]
+        kind = result.get("kind")
+        text = ""
+        attributions: List[Dict[str, Any]] = []
+
+        if kind == "message":
+            text = "\n".join(
+                p.get("text", "")
+                for p in result.get("parts", [])
+                if p.get("kind") == "text"
+            )
+            attributions = result.get("metadata", {}).get("attributions", [])
+        elif kind == "task":
+            state = result.get("status", {}).get("state")
+            if state == "completed":
+                msg = result.get("status", {}).get("message") or {}
+                all_parts = list(msg.get("parts", []))
+                for artifact in result.get("artifacts", []):
+                    all_parts.extend(artifact.get("parts", []))
+                text = "\n".join(
+                    p.get("text", "")
+                    for p in all_parts
+                    if p.get("kind") == "text"
+                )
+                attributions = msg.get("metadata", {}).get("attributions", [])
+            elif state in ("failed", "canceled", "rejected"):
+                status_msg = result.get("status", {}).get("message") or {}
+                detail = "\n".join(
+                    p.get("text", "")
+                    for p in status_msg.get("parts", [])
+                    if p.get("kind") == "text"
+                ).strip()
+                suffix = f" Detail: {detail}" if detail else ""
+                raise RuntimeError(
+                    f"A2A task {state}. Task id: {result.get('id')}{suffix}"
+                )
+            elif state in ("input_required", "auth_required"):
+                requirement = {
+                    "input_required": "user input",
+                    "auth_required": "authentication",
+                }.get(state, state.replace("_", " "))
+                raise RuntimeError(
+                    f"A2A task requires {requirement} and cannot proceed automatically."
+                    f" Task id: {result.get('id')}"
+                )
+            elif state in ("submitted", "working"):
+                raise RuntimeError(
+                    f"A2A task is still {state}; synchronous send returned before completion."
+                    f" Task id: {result.get('id')}"
+                )
+            else:
+                raise RuntimeError(
+                    f"A2A task in unexpected state: {state!r}. Task id: {result.get('id')}"
+                )
+        else:
+            raise RuntimeError(f"Unexpected A2A result kind: {kind!r}")
+
+        if attributions:
+            emit_structured_log(
+                "debug",
+                f"[A2A] Attributions ({len(attributions)}): "
+                + ", ".join(
+                    a.get("providerDisplayName", a.get("attributionType", ""))
+                    for a in attributions
+                ),
+                Operation.SEND_PROMPT,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+
+        display_text = self._replace_citation_markers(text, attributions, self._logger, self._diagnostic_records)
+
+        result_dict = {
+            "raw_response_text": text,
+            "display_response_text": display_text,
+            "a2a_attributions": attributions,
+            "metadata": {
+                "conversation_id": result.get("contextId"),
+            },
+        }
+        return result_dict, result
+
+    @staticmethod
+    def _replace_citation_markers(
+        text: str,
+        attributions: List[Dict[str, Any]],
+        logger: Optional[logging.Logger] = None,
+        diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    ) -> str:
+        """Replace OAI Unicode citation markers with markdown links.
+
+        Marker format: \\ue200cite(\\ue202turn{X}search{Y})+\\ue201
+        Compound markers (multiple turn/search refs between a single pair of
+        bookend characters) are also handled.
+
+        The search{Y} number is NOT a direct array index — it's a grounding result
+        number. Mapping: unique search numbers in first-appearance order →
+        citation_attrs[0, 1, ...].
+
+        Args:
+            text: Response text that may contain OAI citation markers.
+            attributions: Attribution objects from A2A response metadata.
+            logger: Logger for debug messages. Defaults to module logger if not provided.
+            diagnostic_records: List to accumulate structured log entries.
+
+        Returns:
+            Text with citation markers replaced by markdown links, or the
+            original text unchanged if there are no citation attributions.
+        """
+        _logger = logger or logging.getLogger(__name__)
+        citation_attrs = [a for a in attributions if a.get("attributionType") == "citation"]
+        if not text:
+            if text == "" and attributions:
+                emit_structured_log(
+                    "warning",
+                    "[A2A] Response text is empty; skipping citation replacement.",
+                    Operation.SEND_PROMPT,
+                    logger=_logger,
+                    diagnostic_records=diagnostic_records,
+                )
+            return text
+        if not citation_attrs:
+            return text
+
+        # Build ordered map: search-number-string → 0-based index into citation_attrs
+        seen: Dict[str, int] = {}
+        for m in _CITATION_REF_PAT.finditer(text):
+            k = m.group(1)
+            if k not in seen:
+                seen[k] = len(seen)
+
+        def replace_citation(m: re.Match) -> str:
+            links = []
+            for idx_str in _CITATION_REF_PAT.findall(m.group(0)):
+                pos = seen.get(idx_str)
+                if pos is not None and pos < len(citation_attrs):
+                    attr = citation_attrs[pos]
+                    url = attr.get("seeMoreWebUrl") or ""
+                    label = attr.get("providerDisplayName") or url or idx_str
+                    if url:
+                        links.append(f"[{label}]({url})")
+                    else:
+                        emit_structured_log(
+                            "warning",
+                            f"[A2A] Citation search#{idx_str} has no URL; skipping link.",
+                            Operation.SEND_PROMPT,
+                            logger=_logger,
+                            diagnostic_records=diagnostic_records,
+                        )
+                else:
+                    emit_structured_log(
+                        "warning",
+                        f"[A2A] Citation search#{idx_str} has no matching attribution; skipping link.",
+                        Operation.SEND_PROMPT,
+                        logger=_logger,
+                        diagnostic_records=diagnostic_records,
+                    )
+            return " ".join(links)
+
+        return _CITATION_BLOCK_PAT.sub(replace_citation, text)

package/src/clients/cli/api_clients/__init__.py

@@ -0,0 +1,3 @@
+from .base_agent_client import BaseAgentClient
+
+__all__ = ["BaseAgentClient"]

package/src/clients/cli/api_clients/base_agent_client.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import functools
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+import tzlocal
+
+
+class BaseAgentClient(ABC):
+    """Abstract base class for agent API clients.
+    """
+
+    @abstractmethod
+    def fetch_available_agents(self) -> List[Dict[str, Any]]:
+        """Return the list of agents accessible to the configured user.
+
+        Implementations that do not support agent enumeration should
+        return an empty list.
+        """
+        pass
+
+    @abstractmethod
+    def send_prompt(
+        self,
+        prompt: str,
+        agent_id: str | None = None,
+        conversation_context: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+        """Send a single prompt and return the response with conversation context.
+
+        For single-turn usage, pass conversation_context=None.
+        For multi-turn usage, pass the context returned from the previous turn.
+
+        Args:
+            prompt: The prompt string to send.
+            agent_id: Optional agent ID to target.
+            conversation_context: Opaque context dict from a previous turn,
+                or None for the first turn / single-turn usage.
+
+        Returns:
+            Tuple of (enhanced_response_dict, conversation_context).
+            The conversation_context should be passed to the next turn
+            in a multi-turn conversation, or discarded for single-turn.
+            The context structure is implementation-specific:
+              - A2A: {"context_id": str}
+            Returns None as context when no conversation state is established.
+        """
+        pass
+
+    def resolve_agent(self, agent_id: str) -> None:
+        """Pre-resolve agent endpoint. Called once before pipeline starts.
+
+        Default is no-op. Subclasses may override to cache agent discovery.
+        """
+        pass
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_iana_timezone_name() -> str:
+        try:
+            return tzlocal.get_localzone_name()
+        except Exception:
+            return str(tzlocal.get_localzone())
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_location_info() -> Dict[str, Any]:
+        now = datetime.now().astimezone()
+        utc_offset = now.utcoffset()
+        offset_hours = int(utc_offset.total_seconds() // 3600) if utc_offset is not None else 0
+        return {
+            "timeZoneOffset": offset_hours,
+            "timeZone": BaseAgentClient._get_iana_timezone_name(),
+        }
+

package/src/clients/cli/cli_args.py

@@ -0,0 +1,136 @@
+"""CLI argument parsing and version-check bypass logic."""
+
+import argparse
+import os
+
+from cli_logging.cli_logger import emit_structured_log
+from cli_logging.logging_utils import Operation
+from common import MAX_CONCURRENCY, RunConfig
+from agent_selector import normalize_agent_id
+
+
+# Flags that should bypass remote min-version enforcement.
+# --help is not needed here because argparse exits before runtime checks.
+VERSION_CHECK_BYPASS_FLAGS = (
+    "signout",
+)
+
+
+def should_bypass_min_version_check(config: RunConfig) -> bool:
+    """Return True if the current invocation should skip min-version checks."""
+    return any(getattr(config, flag, False) for flag in VERSION_CHECK_BYPASS_FLAGS)
+
+
+def parse_arguments():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="M365 Copilot Agent Evaluation CLI",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run with default prompts
+  python main.py
+
+  # Run with custom prompts
+  python main.py --prompts "What is Microsoft Graph?" --expected "Microsoft Graph is a gateway..."
+
+  # Run with prompts from file
+  python main.py --prompts-file prompts.json
+
+  # Interactive mode
+  python main.py --interactive
+
+  # Save results to JSON
+  python main.py --output results.json
+
+  # Save results to CSV
+  python main.py --output results.csv
+
+  # Save results to HTML and open in browser
+  python main.py --output report.html
+
+  # Debug-level diagnostics
+  python main.py --log-level debug
+
+  # Sign out and clear cached authentication tokens
+  python main.py --signout
+        """
+    )
+    
+    # Input options (mutually exclusive)
+    input_group = parser.add_mutually_exclusive_group()
+    input_group.add_argument(
+        '--prompts', 
+        nargs='+', 
+        help='List of prompts to evaluate'
+    )
+    input_group.add_argument(
+        '--prompts-file', 
+        type=str, 
+        help='JSON file containing prompts and expected responses'
+    )
+    input_group.add_argument(
+        '--interactive', 
+        action='store_true', 
+        help='Interactive mode to enter prompts'
+    )
+    
+    # Expected responses (only used with --prompts)
+    parser.add_argument(
+        '--expected', 
+        nargs='+', 
+        help='List of expected responses (must match number of prompts)'
+    )
+    
+    # Agent ID (--m365-agent-id is primary, --agent-id kept for backward compatibility)
+    parser.add_argument(
+        '--m365-agent-id', '--agent-id',
+        type=str,
+        default=os.environ.get("M365_AGENT_ID") or os.environ.get("AGENT_ID"),
+        help='Agent ID (default from M365_AGENT_ID environment variable)'
+    )
+
+    # Output options
+    parser.add_argument(
+        '--output', 
+        type=str, 
+        help='Output file path. Format is determined by file extension: .json, .csv, .html. If not provided, results are printed to console.'
+    )
+    
+    # Behavior options
+    parser.add_argument(
+        '--log-level',
+        nargs='?',
+        const='info',
+        action='append',
+        help='Set log verbosity: debug, info, warning, error. Bare --log-level resolves to info.'
+    )
+
+    parser.add_argument(
+        '--signout',
+        action='store_true',
+        help='Sign out and clear cached authentication tokens'
+    )
+
+    parser.add_argument(
+        '--concurrency',
+        type=int,
+        default=MAX_CONCURRENCY,
+        help=f'Number of parallel workers for prompt processing (1-{MAX_CONCURRENCY}, default: {MAX_CONCURRENCY})'
+    )
+    
+    args = parser.parse_args()
+
+    args.m365_agent_id = normalize_agent_id(args.m365_agent_id)
+
+    if args.concurrency < 1:
+        parser.error('--concurrency must be an integer >= 1.')
+    if args.concurrency > MAX_CONCURRENCY:
+        emit_structured_log(
+            "warning",
+            f"--concurrency {args.concurrency} exceeds max {MAX_CONCURRENCY}; clamping to {MAX_CONCURRENCY}.",
+            operation=Operation.SETUP,
+        )
+        args.concurrency = MAX_CONCURRENCY
+
+    return args