@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.5.0-preview.1

package/schema/v1/examples/valid/mixed-single-and-multi-turn.json

@@ -0,0 +1,30 @@
+{
+  "schemaVersion": "1.2.0",
+  "default_evaluators": {
+    "Relevance": {},
+    "Coherence": {}
+  },
+  "items": [
+    {
+      "prompt": "What is Microsoft Graph API?",
+      "expected_response": "Microsoft Graph API is a unified endpoint for accessing Microsoft services."
+    },
+    {
+      "name": "Canadian Employee HR Inquiry",
+      "turns": [
+        {
+          "prompt": "I'm a Canadian employee based in Toronto.",
+          "expected_response": "Got it! I can help with Canada-specific HR questions."
+        },
+        {
+          "prompt": "Is July 4th a holiday for me?",
+          "expected_response": "July 4th is not a statutory holiday in Canada. However, July 1st (Canada Day) is."
+        }
+      ]
+    },
+    {
+      "prompt": "How do I authenticate with Microsoft Graph?",
+      "expected_response": "You can authenticate using OAuth 2.0 or client credentials flow."
+    }
+  ]
+}

package/schema/v1/examples/valid/multi-turn-output.json

@@ -0,0 +1,59 @@
+{
+  "schemaVersion": "1.2.0",
+  "metadata": {
+    "evaluatedAt": "2026-03-30T10:00:00Z",
+    "agentName": "Travel Assistant",
+    "cliVersion": "1.3.0"
+  },
+  "items": [
+    {
+      "name": "Context Persistence Test",
+      "turns": [
+        {
+          "prompt": "I'm based in Seattle.",
+          "expected_response": "Got it! I can help with Seattle-specific questions.",
+          "response": "Understood! I can assist with Seattle-related queries.",
+          "scores": {
+            "relevance": {
+              "score": 4.0,
+              "result": "pass",
+              "threshold": 3,
+              "reason": "Response acknowledges Seattle context."
+            },
+            "coherence": {
+              "score": 5.0,
+              "result": "pass",
+              "threshold": 3
+            }
+          },
+          "status": "pass"
+        },
+        {
+          "prompt": "What's the weather like here?",
+          "expected_response": "Seattle weather is typically mild with rain.",
+          "response": "Seattle generally has mild temperatures with frequent rain, especially in fall and winter.",
+          "scores": {
+            "relevance": {
+              "score": 5.0,
+              "result": "pass",
+              "threshold": 3
+            },
+            "coherence": {
+              "score": 4.0,
+              "result": "pass",
+              "threshold": 3
+            }
+          },
+          "status": "pass"
+        }
+      ],
+      "conversation_id": "conv-abc-123",
+      "summary": {
+        "turns_total": 2,
+        "turns_passed": 2,
+        "turns_failed": 0,
+        "overall_status": "pass"
+      }
+    }
+  ]
+}

package/schema/v1/examples/valid/multi-turn-simple.json

@@ -0,0 +1,21 @@
+{
+  "schemaVersion": "1.2.0",
+  "items": [
+    {
+      "turns": [
+        {
+          "prompt": "I'm traveling to Seattle next week for a conference.",
+          "expected_response": "I can help with travel-related questions."
+        },
+        {
+          "prompt": "What's the weather usually like?",
+          "expected_response": "Seattle weather is typically mild with some rain."
+        },
+        {
+          "prompt": "Should I bring a rain jacket?",
+          "expected_response": "Yes, Seattle is known for rain. A rain jacket is recommended."
+        }
+      ]
+    }
+  ]
+}

package/schema/v1/examples/valid/multi-turn-with-evaluators.json

@@ -0,0 +1,34 @@
+{
+  "schemaVersion": "1.2.0",
+  "default_evaluators": {
+    "Relevance": {},
+    "Coherence": {}
+  },
+  "items": [
+    {
+      "name": "Expense Policy Flow",
+      "description": "Test that agent handles expense policy questions across turns",
+      "turns": [
+        {
+          "prompt": "I'm traveling to Seattle next week for a conference.",
+          "expected_response": "I can help with travel-related questions."
+        },
+        {
+          "prompt": "My dinner last night was $250. Is that okay?",
+          "expected_response": "The per-diem meal allowance is a maximum of $200.",
+          "evaluators": {
+            "Groundedness": { "threshold": 4 }
+          }
+        },
+        {
+          "prompt": "What should I do about the overage?",
+          "expected_response": "For expenses exceeding the policy limit, you'll need manager approval.",
+          "evaluators": {
+            "ExactMatch": { "case_sensitive": false }
+          },
+          "evaluators_mode": "replace"
+        }
+      ]
+    }
+  ]
+}

package/schema/version.json

@@ -1,6 +1,6 @@
 {
-  "version": "1.1.0",
-  "releaseDate": "2026-03-17",
+  "version": "1.2.0",
+  "releaseDate": "2026-04-02",
   "schemaId": "https://raw.githubusercontent.com/microsoft/M365-Copilot-Agent-Evals/refs/heads/main/schema/v1/eval-document.schema.json",
   "description": "M365 Copilot Eval Document Schema"
 }

package/src/clients/cli/agent_selector.py

@@ -0,0 +1,74 @@
+"""Interactive agent selection and agent-id utilities."""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import questionary
+
+from cli_logging.cli_logger import emit_structured_log
+from cli_logging.logging_utils import Operation
+
+
+def normalize_agent_id(agent_id):
+    """Append '.declarativeAgent' if agent_id has no '.', else return unchanged.
+
+    Returns the input unchanged when it is None/empty or already contains a dot.
+    """
+    if not agent_id:
+        return agent_id
+    return agent_id if '.' in agent_id else f"{agent_id}.declarativeAgent"
+
+
+def select_agent_interactively(agents: List[Dict[str, Any]]) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Display an interactive agent selector using questionary.
+
+    Args:
+        agents: List of agent dictionaries.
+
+    Returns:
+        Tuple of (agent_id, agent_name) or (None, None) if cancelled/skipped
+    """
+    if not agents:
+        return None, None
+
+    # Build id→name lookup and choices
+    id_to_name: Dict[str, str] = {}
+    choices = []
+    sorted_agents = sorted(agents, key=lambda a: a.get("name", ""))
+    for agent in sorted_agents:
+        agent_name = agent.get("name", "Unknown")
+        agent_id = (agent.get("gptId") or "").strip()
+        if not agent_id:
+            emit_structured_log("warning", f"Skipping agent '{agent_name}': missing or empty gptId.", operation=Operation.FETCH_AGENTS)
+            continue
+        agent_description = agent.get("description")
+        agent_is_owner = agent.get('isOwner')
+        agent_provider = agent.get("provider")
+        id_to_name[agent_id] = agent_name
+
+        # Format the display text
+        if agent_provider:
+            title = f"{agent_name} - {agent_provider} ({agent_id})"
+        else:
+            title = f"{agent_name} ({agent_id})"
+        segments = [title]
+        if agent_is_owner:
+            segments.append(f"IsOwner: {agent_is_owner}")
+        if agent_description:
+            segments.append(agent_description)
+        display_text = " - ".join(segments)
+
+        choices.append(questionary.Choice(title=display_text, value=agent_id))
+
+    if not choices:
+        return None, None
+
+    # Display the selection prompt
+    selected_agent = questionary.select(
+        "Select an agent to evaluate:",
+        choices=choices,
+        use_shortcuts=len(choices) <= 35,
+        use_arrow_keys=True
+    ).ask()
+
+    return selected_agent, id_to_name.get(selected_agent) if selected_agent else None

package/src/clients/cli/api_clients/A2A/__init__.py

@@ -0,0 +1,3 @@
+from .a2a_client import A2AClient
+
+__all__ = ["A2AClient"]