@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.4.0-preview.1

package/src/clients/cli/api_clients/REST/sydney_client.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Tuple
+
+from api_clients.base_agent_client import BaseAgentClient
+from cli_logging.console_diagnostics import emit_structured_log
+from cli_logging.logging_utils import Operation
+from response_extractor import extract_enhanced_response
+
+_REQUEST_TIMEOUT_SECS = 120
+
+# int → str used when passing log level to extract_enhanced_response
+_LEVEL_INT_TO_STR: Dict[int, str] = {
+    logging.DEBUG: "debug",
+    logging.INFO: "info",
+    logging.WARNING: "warning",
+    logging.ERROR: "error",
+}
+
+
+class SydneyClient(BaseAgentClient):
+    """REST client for the Microsoft Copilot Sydney chat API."""
+
+    def __init__(
+        self,
+        *,
+        copilot_api_endpoint: str,
+        access_token: str,
+        user_oid: str,
+        logger: Optional[logging.Logger] = None,
+        diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """
+        Args:
+            copilot_api_endpoint: Base URL for the Copilot API.
+            access_token: Bearer token for API authentication.
+            user_oid: User object ID included in request payloads.
+            logger: Logger to use for all client logging. Defaults to a module-level
+                logger if not provided.
+            diagnostic_records: List to accumulate structured log entries.
+        """
+        self._endpoint = copilot_api_endpoint
+        self._access_token = access_token
+        self._user_oid = user_oid
+        self._logger = logger or logging.getLogger(__name__)
+        self._diagnostic_records = diagnostic_records
+        self._log_level = _LEVEL_INT_TO_STR.get(self._logger.getEffectiveLevel(), "info")
+
+    # ------------------------------------------------------------------ #
+    #  BaseAgentClient implementation                                     #
+    # ------------------------------------------------------------------ #
+
+    def fetch_available_agents(self) -> List[Dict[str, Any]]:
+        """Fetch agents available to the user from the Copilot API.
+
+        Returns an empty list if the endpoint is unavailable or returns an error.
+        """
+        try:
+            request_data = json.dumps({"participant": {"id": self._user_oid}})
+            query_param = urllib.parse.quote(request_data)
+            agents_url = f"{self._endpoint}/GetGptList?request={query_param}"
+            emit_structured_log(
+                "debug",
+                f"[REST] Fetching available agents from: {agents_url}",
+                Operation.FETCH_AGENTS,
+                logger=self._logger,
+                diagnostic_records=self._diagnostic_records,
+            )
+            req = urllib.request.Request(
+                agents_url,
+                headers=self._build_request_headers(),
+                method="GET",
+            )
+            with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+                return data.get("gptList", [])
+        except urllib.error.HTTPError as e:
+            emit_structured_log("warning", f"[REST] Unable to fetch agents list (HTTP {e.code}).", Operation.FETCH_AGENTS, logger=self._logger, diagnostic_records=self._diagnostic_records)
+            return []
+        except Exception as e:
+            emit_structured_log("warning", f"[REST] Error fetching agents: {e}", Operation.FETCH_AGENTS, logger=self._logger, diagnostic_records=self._diagnostic_records)
+            return []
+
+    def send_prompt(
+        self,
+        prompt: str,
+        agent_id: str | None = None,
+        conversation_context: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+        """Send a prompt to the Sydney /chat endpoint and return the response with context.
+
+        Args:
+            prompt: Prompt string to send to the agent.
+            agent_id: Optional agent ID to target a specific Copilot agent.
+            conversation_context: Context from a previous turn (contains conversation_id),
+                or None for the first turn / single-turn usage.
+
+        Returns:
+            Tuple of (enhanced_response_dict, conversation_context).
+        """
+        request_headers = self._build_request_headers()
+        conversation_id = conversation_context.get("conversation_id") if conversation_context else None
+
+        emit_structured_log("debug", "[REST] Sending prompt to agent.", Operation.SEND_PROMPT, logger=self._logger, diagnostic_records=self._diagnostic_records)
+
+        payload = self._build_chat_payload(prompt, agent_id, conversation_id)
+        emit_structured_log("debug", f"[REST] Sending payload: {payload.decode('utf-8')[:500]}", Operation.SEND_PROMPT, logger=self._logger, diagnostic_records=self._diagnostic_records)
+
+        req = urllib.request.Request(
+            f"{self._endpoint}/chat",
+            data=payload,
+            headers=request_headers,
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT_SECS) as resp:
+                raw = resp.read().decode("utf-8", errors="replace")
+        except urllib.error.HTTPError as e:
+            error_body = None
+            try:
+                error_body = e.read().decode("utf-8", errors="replace")
+            except Exception:
+                pass
+            msg = f"[REST] Chat API request failed (HTTP {e.code} {e.reason})."
+            if error_body:
+                msg += f" Body: {error_body[:500]}"
+            raise RuntimeError(msg) from e
+        except urllib.error.URLError as e:
+            raise RuntimeError(
+                f"[REST] Chat API connection error: {getattr(e, 'reason', str(e))}"
+            ) from e
+
+        emit_structured_log("debug", f"[REST] Raw response: {raw[:500]}", Operation.SEND_PROMPT, logger=self._logger, diagnostic_records=self._diagnostic_records)
+
+        enhanced_response = extract_enhanced_response(raw.strip(), self._log_level)
+
+        metadata = enhanced_response.get("metadata", {})
+        emit_structured_log(
+            "debug",
+            "Response IDs for prompt.",
+            Operation.SEND_PROMPT,
+            logger=self._logger,
+            diagnostic_records=self._diagnostic_records,
+            run_context={
+                "operation": Operation.SEND_PROMPT,
+                "request-id": metadata.get("request_id"),
+                "conversation-id": metadata.get("conversation_id"),
+                "message-id": metadata.get("message_id"),
+            },
+        )
+
+        # Build updated context for subsequent turns
+        new_conversation_id = metadata.get("conversation_id") or conversation_id
+        updated_context = {"conversation_id": new_conversation_id} if new_conversation_id else None
+
+        return enhanced_response, updated_context
+
+    # ------------------------------------------------------------------ #
+    #  Private helpers                                                    #
+    # ------------------------------------------------------------------ #
+
+    def _build_request_headers(self) -> Dict[str, str]:
+        headers = {
+            "Content-Type": "application/json",
+            "X-Scenario": os.environ.get("X_SCENARIO_HEADER"),
+            "Authorization": f"Bearer {self._access_token}",
+        }
+        return {k: v for k, v in headers.items() if v is not None}
+
+    def _build_chat_payload(
+        self,
+        prompt: str,
+        agent_id: str | None,
+        conversation_id: str | None = None,
+    ) -> bytes:
+        message: Dict[str, Any] = {
+            "message": {
+                "text": prompt,
+                "author": "user",
+                "messageType": "chat",
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "locationInfo": self._get_location_info(),
+                "from": {
+                    "id": self._user_oid,
+                },
+            },
+            "verbosity": "verbose",
+        }
+
+        if agent_id:
+            message["gpts"] = [{"id": agent_id.strip(), "source": "MOS3"}]
+            message["optionsSets"] = ["disable_action_confirmation"]
+
+        if conversation_id:
+            message["conversationId"] = conversation_id
+
+        return json.dumps(message).encode("utf-8")

package/src/clients/cli/api_clients/__init__.py

@@ -0,0 +1,3 @@
+from .base_agent_client import BaseAgentClient
+
+__all__ = ["BaseAgentClient"]

package/src/clients/cli/api_clients/base_agent_client.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import functools
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+import tzlocal
+
+
+class BaseAgentClient(ABC):
+    """Abstract base class for agent API clients.
+    """
+
+    @abstractmethod
+    def fetch_available_agents(self) -> List[Dict[str, Any]]:
+        """Return the list of agents accessible to the configured user.
+
+        Implementations that do not support agent enumeration should
+        return an empty list.
+        """
+        pass
+
+    @abstractmethod
+    def send_prompt(
+        self,
+        prompt: str,
+        agent_id: str | None = None,
+        conversation_context: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+        """Send a single prompt and return the response with conversation context.
+
+        For single-turn usage, pass conversation_context=None.
+        For multi-turn usage, pass the context returned from the previous turn.
+
+        Args:
+            prompt: The prompt string to send.
+            agent_id: Optional agent ID to target.
+            conversation_context: Opaque context dict from a previous turn,
+                or None for the first turn / single-turn usage.
+
+        Returns:
+            Tuple of (enhanced_response_dict, conversation_context).
+            The conversation_context should be passed to the next turn
+            in a multi-turn conversation, or discarded for single-turn.
+            The context structure is implementation-specific:
+              - Sydney/REST: {"conversation_id": str}
+              - A2A: {"context_id": str}
+            Returns None as context when no conversation state is established.
+        """
+        pass
+
+    def resolve_agent(self, agent_id: str) -> None:
+        """Pre-resolve agent endpoint. Called once before pipeline starts.
+
+        Default is no-op. Subclasses may override to cache agent discovery.
+        """
+        pass
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_iana_timezone_name() -> str:
+        try:
+            return tzlocal.get_localzone_name()
+        except Exception:
+            return str(tzlocal.get_localzone())
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_location_info() -> Dict[str, Any]:
+        now = datetime.now().astimezone()
+        utc_offset = now.utcoffset()
+        offset_hours = int(utc_offset.total_seconds() // 3600) if utc_offset is not None else 0
+        return {
+            "timeZoneOffset": offset_hours,
+            "timeZone": BaseAgentClient._get_iana_timezone_name(),
+        }
+

package/src/clients/cli/cli_logging/console_diagnostics.py

@@ -1,9 +1,14 @@
 import json
+import logging
 import sys
 from collections import OrderedDict
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
 
-from cli_logging.logging_utils import STRUCTURED_LOG_FIELDS
+from cli_logging.logging_utils import (
+    STRUCTURED_LOG_FIELDS,
+    Operation,
+    format_structured_log_entry,
+)
 
 _ANSI_COLORS = {
     "debug": "\033[2m",     # dim
@@ -53,3 +58,50 @@ def render_diagnostic(record: Dict[str, Any]) -> str:
     if sys.stdout.isatty():
         return format_console_record(record)
     return serialize_diagnostic_record(record)
+
+
+def emit_structured_log(
+    level: str,
+    message: str,
+    operation: str = Operation.EVALUATE,
+    *,
+    logger: logging.Logger,
+    diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    run_context: Optional[Dict[str, Any]] = None,
+) -> None:
+    """Emit a structured log entry.
+
+    Formats via format_structured_log_entry, optionally appends to
+    diagnostic_records, then logs via render_diagnostic (TTY-friendly or JSON).
+
+    Args:
+        level: One of "debug", "info", "warning", "error".
+        message: Human-readable log message.
+        operation: The CLI operation step (e.g. Operation.SEND_PROMPT).
+        logger: Logger to emit through.
+        diagnostic_records: If provided, the structured entry is appended here.
+        run_context: Full run context override (request-id, conversation-id,
+            message-id). Defaults to nulls with the given operation.
+    """
+    log_level_int = getattr(logging, level.upper(), logging.INFO)
+    if diagnostic_records is None and not logger.isEnabledFor(log_level_int):
+        return
+
+    context = run_context or {
+        "request-id": None,
+        "conversation-id": None,
+        "message-id": None,
+        "operation": operation,
+    }
+    entry = format_structured_log_entry(
+        level=level,
+        message=message,
+        logger_name=logger.name,
+        run_context=context,
+    )
+    if diagnostic_records is not None:
+        diagnostic_records.append(entry)
+    try:
+        logger.log(getattr(logging, level.upper(), logging.INFO), render_diagnostic(entry))
+    except Exception:
+        pass

package/src/clients/cli/cli_logging/logging_utils.py

@@ -52,7 +52,6 @@ def normalize_log_level(value: Optional[str]) -> Optional[str]:
     return value.strip().lower()
 
 
-
 def resolve_log_level(
     log_level_values: Optional[List[str]],
 ) -> Tuple[Optional[str], Optional[str]]:

package/src/clients/cli/common.py

@@ -25,6 +25,17 @@ PARTIAL_MATCH = "PartialMatch"
 REQUIRES_AZURE_OPENAI = "azure_openai"
 REQUIRES_TOOL_DEFINITIONS = "tool_definitions"
 
+# Evaluation status constants
+# Outcome statuses (agent responded, evaluators ran):
+STATUS_PASS = "pass"  # All evaluators scored above threshold
+STATUS_FAIL = "fail"  # At least one evaluator scored below threshold
+# Error state (evaluation couldn't complete):
+STATUS_ERROR = "error"  # API call failed / response couldn't be obtained
+# Thread-level aggregate status (multi-turn only):
+STATUS_PARTIAL = "partial"  # Some turns passed, some did not
+# Fallback for missing status:
+STATUS_UNKNOWN = "unknown"
+
 # System defaults when no file-level or env-level defaults are configured
 SYSTEM_DEFAULT_EVALUATORS = [
     RELEVANCE,