@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.4.0-preview.1

package/schema/v1/examples/valid/multi-turn-output.json

@@ -0,0 +1,59 @@
+{
+  "schemaVersion": "1.2.0",
+  "metadata": {
+    "evaluatedAt": "2026-03-30T10:00:00Z",
+    "agentName": "Travel Assistant",
+    "cliVersion": "1.3.0"
+  },
+  "items": [
+    {
+      "name": "Context Persistence Test",
+      "turns": [
+        {
+          "prompt": "I'm based in Seattle.",
+          "expected_response": "Got it! I can help with Seattle-specific questions.",
+          "response": "Understood! I can assist with Seattle-related queries.",
+          "scores": {
+            "relevance": {
+              "score": 4.0,
+              "result": "pass",
+              "threshold": 3,
+              "reason": "Response acknowledges Seattle context."
+            },
+            "coherence": {
+              "score": 5.0,
+              "result": "pass",
+              "threshold": 3
+            }
+          },
+          "status": "pass"
+        },
+        {
+          "prompt": "What's the weather like here?",
+          "expected_response": "Seattle weather is typically mild with rain.",
+          "response": "Seattle generally has mild temperatures with frequent rain, especially in fall and winter.",
+          "scores": {
+            "relevance": {
+              "score": 5.0,
+              "result": "pass",
+              "threshold": 3
+            },
+            "coherence": {
+              "score": 4.0,
+              "result": "pass",
+              "threshold": 3
+            }
+          },
+          "status": "pass"
+        }
+      ],
+      "conversation_id": "conv-abc-123",
+      "summary": {
+        "turns_total": 2,
+        "turns_passed": 2,
+        "turns_failed": 0,
+        "overall_status": "pass"
+      }
+    }
+  ]
+}

package/schema/v1/examples/valid/multi-turn-simple.json

@@ -0,0 +1,21 @@
+{
+  "schemaVersion": "1.2.0",
+  "items": [
+    {
+      "turns": [
+        {
+          "prompt": "I'm traveling to Seattle next week for a conference.",
+          "expected_response": "I can help with travel-related questions."
+        },
+        {
+          "prompt": "What's the weather usually like?",
+          "expected_response": "Seattle weather is typically mild with some rain."
+        },
+        {
+          "prompt": "Should I bring a rain jacket?",
+          "expected_response": "Yes, Seattle is known for rain. A rain jacket is recommended."
+        }
+      ]
+    }
+  ]
+}

package/schema/v1/examples/valid/multi-turn-with-evaluators.json

@@ -0,0 +1,34 @@
+{
+  "schemaVersion": "1.2.0",
+  "default_evaluators": {
+    "Relevance": {},
+    "Coherence": {}
+  },
+  "items": [
+    {
+      "name": "Expense Policy Flow",
+      "description": "Test that agent handles expense policy questions across turns",
+      "turns": [
+        {
+          "prompt": "I'm traveling to Seattle next week for a conference.",
+          "expected_response": "I can help with travel-related questions."
+        },
+        {
+          "prompt": "My dinner last night was $250. Is that okay?",
+          "expected_response": "The per-diem meal allowance is a maximum of $200.",
+          "evaluators": {
+            "Groundedness": { "threshold": 4 }
+          }
+        },
+        {
+          "prompt": "What should I do about the overage?",
+          "expected_response": "For expenses exceeding the policy limit, you'll need manager approval.",
+          "evaluators": {
+            "ExactMatch": { "case_sensitive": false }
+          },
+          "evaluators_mode": "replace"
+        }
+      ]
+    }
+  ]
+}

package/schema/version.json

@@ -1,6 +1,6 @@
 {
-  "version": "1.1.0",
-  "releaseDate": "2026-03-17",
+  "version": "1.2.0",
+  "releaseDate": "2026-04-02",
   "schemaId": "https://raw.githubusercontent.com/microsoft/M365-Copilot-Agent-Evals/refs/heads/main/schema/v1/eval-document.schema.json",
   "description": "M365 Copilot Eval Document Schema"
 }

package/src/clients/cli/api_clients/A2A/__init__.py

@@ -0,0 +1,3 @@
+from .a2a_client import A2AClient
+
+__all__ = ["A2AClient"]

package/src/clients/cli/api_clients/A2A/a2a_client.py

@@ -0,0 +1,456 @@
+from __future__ import annotations
+
+import functools
+import json
+import locale
+import logging
+import re
+import urllib.error
+import urllib.request
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+from api_clients.base_agent_client import BaseAgentClient
+from cli_logging.console_diagnostics import emit_structured_log
+from cli_logging.logging_utils import Operation
+
+# Feature flag required by the experimental A2A surface.
+_A2A_FEATURE_FLAG = "feature.EnableA2AServer"
+
+_REQUEST_TIMEOUT_SECS = 120
+
+# OAI citation marker patterns — compiled once at module level.
+# Marker format: \ue200cite(\ue202turn{X}search{Y})+\ue201
+_CITATION_REF_PAT = re.compile(r"\ue202turn\d+search(\d+)")
+_CITATION_BLOCK_PAT = re.compile(r"\ue200cite(?:\ue202turn\d+search\d+)+\ue201")
+
+
+class A2AClient(BaseAgentClient):
+    """A2A (Agent-to-Agent) JSON-RPC 2.0 client for Work IQ agents."""
+
+    def __init__(
+        self,
+        *,
+        a2a_endpoint: str,
+        access_token: str,
+        logger: Optional[logging.Logger] = None,
+        diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """
+        Args:
+            a2a_endpoint: Base URL of the A2A endpoint.
+            access_token: Bearer token for A2A authentication.
+            logger: Logger to use. Defaults to a module-level logger if not provided.
+            diagnostic_records: List to accumulate structured log entries.
+        """
+        self._endpoint = a2a_endpoint.rstrip("/")
+        self._access_token = access_token
+        self._logger = logger or logging.getLogger(__name__)
+        self._diagnostic_records = diagnostic_records
+        self._resolved_agent_url: Optional[str] = None
+
+    # ------------------------------------------------------------------ #
+    #  BaseAgentClient implementation                                     #
+    # ------------------------------------------------------------------ #
+
+    def resolve_agent(self, agent_id: str) -> None:
+        """Pre-resolve agent URL from agent card and cache for the session."""
+        self._resolved_agent_url = self._resolve_agent_url(agent_id)
+
+    def fetch_available_agents(self) -> List[Dict[str, Any]]:
+        """Fetch agents from the A2A discovery endpoint.
+
+        Calls GET {endpoint}/.agents. Each A2A agent card is normalized to
+        include 'gptId', 'name', and 'description' so it is compatible with
+        the shared select_agent_interactively selector.
+
+        Returns an empty list if the endpoint is unreachable or returns an
+        error.
+        """
+        try:
+            agents_url = f"{self._endpoint}/.agents"
+            headers = self._build_request_headers()
+            emit_structured_log(
+                "debug",
+                f"[A2A] Fetching available agents from: {agents_url}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            req = urllib.request.Request(agents_url, headers=headers, method="GET")
+            with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+            agents = data if isinstance(data, list) else data.get("agents", [])
+            return [self._normalize_agent_card(a) for a in agents]
+        except urllib.error.HTTPError as e:
+            emit_structured_log(
+                "warning",
+                f"[A2A] Unable to fetch agents list (HTTP {e.code}).",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            return []
+        except Exception as e:
+            emit_structured_log(
+                "warning",
+                f"[A2A] Error fetching agents: {e}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            return []
+
+    @staticmethod
+    def _normalize_agent_card(agent: Dict[str, Any]) -> Dict[str, Any]:
+        """Normalize an A2A agent card to the shape expected by the selector.
+
+        A2A agent cards use a 'url' field rather than a discrete ID.  The
+        agent ID is extracted as the last path segment of that URL, falling
+        back to the agent name when the URL is absent.
+        """
+        agent_url = agent.get("url", "")
+        agent_id = (
+            agent_url.rstrip("/").rsplit("/", 1)[-1]
+            if agent_url
+            else agent.get("name", "")
+        )
+        return {
+            "gptId": agent_id,
+            "name": agent.get("name", agent_id),
+            "description": agent.get("description", ""),
+            "isOwner": False,
+        }
+
+    def send_prompt(
+        self,
+        prompt: str,
+        agent_id: str | None = None,
+        conversation_context: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+        """Send a single prompt to the A2A endpoint and return the response with context.
+
+        Args:
+            prompt: The prompt string to send.
+            agent_id: Target agent ID. Required for A2A — no fallback discovery.
+            conversation_context: Context from a previous turn (contains context_id),
+                or None for the first turn / single-turn usage.
+
+        Returns:
+            Tuple of (enhanced_response_dict, conversation_context).
+        """
+        agent_id = (agent_id or "").strip()
+        if not agent_id:
+            raise ValueError("agent_id is required for A2A requests.")
+
+        headers = self._build_request_headers(include_content_type=True)
+        agent_url = self._resolved_agent_url or self._resolve_agent_url(agent_id)
+
+        context_id = conversation_context.get("context_id") if conversation_context else None
+
+        emit_structured_log(
+            "debug",
+            "[A2A] Sending prompt to agent.",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+
+        payload = self._build_chat_payload(prompt, context_id)
+
+        emit_structured_log(
+            "debug",
+            f"[A2A] Sending to {agent_url}: {payload.decode('utf-8')[:500]}",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+
+        result_dict, raw_result = self._send_and_parse_message(agent_url, payload, headers)
+
+        # Build updated context for subsequent turns
+        new_context_id = context_id
+        if raw_result:
+            new_context_id = raw_result.get("contextId") or context_id
+        updated_context = {"context_id": new_context_id} if new_context_id else None
+
+        return result_dict, updated_context
+
+    # ------------------------------------------------------------------ #
+    #  Private helpers                                                    #
+    # ------------------------------------------------------------------ #
+
+    def _build_request_headers(self, *, include_content_type: bool = False) -> Dict[str, str]:
+        headers: Dict[str, str] = {
+            "Authorization": f"Bearer {self._access_token}",
+            "X-variants": _A2A_FEATURE_FLAG,
+        }
+        if include_content_type:
+            headers["Content-Type"] = "application/json"
+        return headers
+
+    def _build_chat_payload(self, prompt: str, context_id: str | None = None) -> bytes:
+        message: Dict[str, Any] = {
+            "kind": "message",
+            "role": "user",
+            "parts": [{"kind": "text", "text": prompt}],
+            "messageId": str(uuid.uuid4()),
+            "metadata": {
+                "location": self._get_a2a_location(),
+            },
+        }
+        if context_id:
+            message["contextId"] = context_id
+        return json.dumps({
+            "jsonrpc": "2.0",
+            "method": "message/send",
+            "params": {"message": message},
+            "id": str(uuid.uuid4()),
+        }).encode("utf-8")
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_a2a_location() -> Dict[str, Any]:
+        locale_str = locale.getlocale()[0] or ""
+        country = locale_str.split("_")[-1] if "_" in locale_str else ""
+        return {
+            "countryOrRegion": country,
+            "countryOrRegionConfidence": 1.0,
+            "timeZone": BaseAgentClient._get_iana_timezone_name(),
+        }
+
+    def _resolve_agent_url(self, agent_id: str) -> str:
+        """Resolve the agent URL from the agent card, falling back to base URL.
+
+        """
+        headers = self._build_request_headers(include_content_type=True)
+        base_agent_url = f"{self._endpoint}/{agent_id}"
+        card_url = f"{base_agent_url}/.well-known/agent-card.json"
+        agent_url = base_agent_url
+        emit_structured_log(
+            "debug",
+            f"[A2A] Fetching agent card from: {card_url}",
+            Operation.FETCH_AGENTS,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+        try:
+            card_req = urllib.request.Request(card_url, headers=headers)
+            with urllib.request.urlopen(card_req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                raw_card = resp.read().decode("utf-8")
+            if raw_card.strip():
+                card = json.loads(raw_card)
+                agent_url = card.get("url") or base_agent_url
+        except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError, UnicodeDecodeError) as e:
+            emit_structured_log(
+                "debug",
+                f"[A2A] Agent card fetch failed ({e}); using base URL: {base_agent_url}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+        emit_structured_log(
+            "debug",
+            f"[A2A] Resolved agent URL: {agent_url}",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+        return agent_url
+
+    def _send_and_parse_message(
+        self,
+        agent_url: str,
+        payload: bytes,
+        headers: Dict[str, str],
+    ) -> tuple[Dict[str, Any], Dict[str, Any]]:
+        """Send a JSON-RPC message to the agent and parse the response.
+
+        Returns:
+            A tuple of (result_dict, raw_result) where result_dict is the
+            normalized response dict (raw_response_text, display_response_text,
+            a2a_attributions) and raw_result is the parsed JSON result object.
+
+        Raises:
+            RuntimeError: On HTTP errors, connection errors, JSON parse errors,
+                or A2A protocol errors.
+        """
+        req = urllib.request.Request(agent_url, data=payload, headers=headers, method="POST")
+        try:
+            with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                raw = resp.read().decode("utf-8", errors="replace")
+        except urllib.error.HTTPError as e:
+            body = ""
+            try:
+                body = e.read().decode("utf-8", errors="replace")
+            except Exception:
+                pass
+            raise RuntimeError(
+                f"A2A request failed (HTTP {e.code} {e.reason})."
+                + (f" Body: {body[:500]}" if body else "")
+            ) from e
+        except urllib.error.URLError as e:
+            raise RuntimeError(
+                f"A2A connection error: {getattr(e, 'reason', str(e))}"
+            ) from e
+
+        emit_structured_log(
+            "debug",
+            f"[A2A] Raw response: {raw[:500]}",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+        )
+
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as e:
+            raise RuntimeError(f"A2A response is not valid JSON: {e}") from e
+
+        if "error" in data:
+            err = data["error"]
+            raise RuntimeError(
+                f"A2A JSON-RPC error {err.get('code', 'unknown')}: {err.get('message', 'no message')}"
+            )
+
+        if "result" not in data:
+            raise RuntimeError(
+                f"A2A response missing 'result' key. Keys present: {list(data.keys())}"
+            )
+
+        result = data["result"]
+        kind = result.get("kind")
+        text = ""
+        attributions: List[Dict[str, Any]] = []
+
+        if kind == "message":
+            text = "\n".join(
+                p.get("text", "")
+                for p in result.get("parts", [])
+                if p.get("kind") == "text"
+            )
+            attributions = result.get("metadata", {}).get("attributions", [])
+        elif kind == "task":
+            state = result.get("status", {}).get("state")
+            if state == "completed":
+                msg = result.get("status", {}).get("message") or {}
+                text = "\n".join(
+                    p.get("text", "")
+                    for p in msg.get("parts", [])
+                    if p.get("kind") == "text"
+                )
+                attributions = msg.get("metadata", {}).get("attributions", [])
+            elif state in ("failed", "canceled"):
+                raise RuntimeError(
+                    f"A2A task {state}. Task id: {result.get('id')}"
+                )
+            else:
+                raise RuntimeError(
+                    f"A2A task in unexpected state: {state!r}. Task id: {result.get('id')}"
+                )
+        else:
+            raise RuntimeError(f"Unexpected A2A result kind: {kind!r}")
+
+        if attributions:
+            emit_structured_log(
+                "debug",
+                f"[A2A] Attributions ({len(attributions)}): "
+                + ", ".join(
+                    a.get("providerDisplayName", a.get("attributionType", ""))
+                    for a in attributions
+                ),
+                Operation.SEND_PROMPT,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+
+        display_text = self._replace_citation_markers(text, attributions, self._logger, self._diagnostic_records)
+
+        result_dict = {
+            "raw_response_text": text,
+            "display_response_text": display_text,
+            "a2a_attributions": attributions,
+            "metadata": {
+                "conversation_id": result.get("contextId"),
+            },
+        }
+        return result_dict, result
+
+    @staticmethod
+    def _replace_citation_markers(
+        text: str,
+        attributions: List[Dict[str, Any]],
+        logger: Optional[logging.Logger] = None,
+        diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    ) -> str:
+        """Replace OAI Unicode citation markers with markdown links.
+
+        Marker format: \\ue200cite(\\ue202turn{X}search{Y})+\\ue201
+        Compound markers (multiple turn/search refs between a single pair of
+        bookend characters) are also handled.
+
+        The search{Y} number is NOT a direct array index — it's a grounding result
+        number. Mapping: unique search numbers in first-appearance order →
+        citation_attrs[0, 1, ...].
+
+        Args:
+            text: Response text that may contain OAI citation markers.
+            attributions: Attribution objects from A2A response metadata.
+            logger: Logger for debug messages. Defaults to module logger if not provided.
+            diagnostic_records: List to accumulate structured log entries.
+
+        Returns:
+            Text with citation markers replaced by markdown links, or the
+            original text unchanged if there are no citation attributions.
+        """
+        _logger = logger or logging.getLogger(__name__)
+        citation_attrs = [a for a in attributions if a.get("attributionType") == "citation"]
+        if not text:
+            if text == "" and attributions:
+                emit_structured_log(
+                    "warning",
+                    "[A2A] Response text is empty; skipping citation replacement.",
+                    Operation.SEND_PROMPT,
+                    logger=_logger,
+                    diagnostic_records=diagnostic_records,
+                )
+            return text
+        if not citation_attrs:
+            return text
+
+        # Build ordered map: search-number-string → 0-based index into citation_attrs
+        seen: Dict[str, int] = {}
+        for m in _CITATION_REF_PAT.finditer(text):
+            k = m.group(1)
+            if k not in seen:
+                seen[k] = len(seen)
+
+        def replace_citation(m: re.Match) -> str:
+            links = []
+            for idx_str in _CITATION_REF_PAT.findall(m.group(0)):
+                pos = seen.get(idx_str)
+                if pos is not None and pos < len(citation_attrs):
+                    attr = citation_attrs[pos]
+                    url = attr.get("seeMoreWebUrl") or ""
+                    label = attr.get("providerDisplayName") or url or idx_str
+                    if url:
+                        links.append(f"[{label}]({url})")
+                    else:
+                        emit_structured_log(
+                            "warning",
+                            f"[A2A] Citation search#{idx_str} has no URL; skipping link.",
+                            Operation.SEND_PROMPT,
+                            logger=_logger,
+                            diagnostic_records=diagnostic_records,
+                        )
+                else:
+                    emit_structured_log(
+                        "warning",
+                        f"[A2A] Citation search#{idx_str} has no matching attribution; skipping link.",
+                        Operation.SEND_PROMPT,
+                        logger=_logger,
+                        diagnostic_records=diagnostic_records,
+                    )
+            return " ".join(links)
+
+        return _CITATION_BLOCK_PAT.sub(replace_citation, text)

package/src/clients/cli/api_clients/REST/__init__.py

@@ -0,0 +1,3 @@
+from .sydney_client import SydneyClient
+
+__all__ = ["SydneyClient"]