@microsoft/m365-copilot-eval 1.2.1-preview.1 → 1.4.0-preview.1

package/src/clients/cli/api_clients/base_agent_client.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import functools
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+import tzlocal
+
+
+class BaseAgentClient(ABC):
+    """Abstract base class for agent API clients.
+    """
+
+    @abstractmethod
+    def fetch_available_agents(self) -> List[Dict[str, Any]]:
+        """Return the list of agents accessible to the configured user.
+
+        Implementations that do not support agent enumeration should
+        return an empty list.
+        """
+        pass
+
+    @abstractmethod
+    def send_prompt(
+        self,
+        prompt: str,
+        agent_id: str | None = None,
+        conversation_context: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+        """Send a single prompt and return the response with conversation context.
+
+        For single-turn usage, pass conversation_context=None.
+        For multi-turn usage, pass the context returned from the previous turn.
+
+        Args:
+            prompt: The prompt string to send.
+            agent_id: Optional agent ID to target.
+            conversation_context: Opaque context dict from a previous turn,
+                or None for the first turn / single-turn usage.
+
+        Returns:
+            Tuple of (enhanced_response_dict, conversation_context).
+            The conversation_context should be passed to the next turn
+            in a multi-turn conversation, or discarded for single-turn.
+            The context structure is implementation-specific:
+              - Sydney/REST: {"conversation_id": str}
+              - A2A: {"context_id": str}
+            Returns None as context when no conversation state is established.
+        """
+        pass
+
+    def resolve_agent(self, agent_id: str) -> None:
+        """Pre-resolve agent endpoint. Called once before pipeline starts.
+
+        Default is no-op. Subclasses may override to cache agent discovery.
+        """
+        pass
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_iana_timezone_name() -> str:
+        try:
+            return tzlocal.get_localzone_name()
+        except Exception:
+            return str(tzlocal.get_localzone())
+
+    @staticmethod
+    @functools.lru_cache(maxsize=1)
+    def _get_location_info() -> Dict[str, Any]:
+        now = datetime.now().astimezone()
+        utc_offset = now.utcoffset()
+        offset_hours = int(utc_offset.total_seconds() // 3600) if utc_offset is not None else 0
+        return {
+            "timeZoneOffset": offset_hours,
+            "timeZone": BaseAgentClient._get_iana_timezone_name(),
+        }
+

package/src/clients/cli/cli_logging/console_diagnostics.py

@@ -0,0 +1,107 @@
+import json
+import logging
+import sys
+from collections import OrderedDict
+from typing import Any, Dict, List, Optional
+
+from cli_logging.logging_utils import (
+    STRUCTURED_LOG_FIELDS,
+    Operation,
+    format_structured_log_entry,
+)
+
+_ANSI_COLORS = {
+    "debug": "\033[2m",     # dim
+    "info": "",             # default
+    "warning": "\033[33m",  # yellow
+    "error": "\033[31m",    # red
+}
+_ANSI_RESET = "\033[0m"
+
+
+def format_diagnostic_record(record: Dict[str, Any]) -> OrderedDict:
+    ordered = OrderedDict()
+    for field in STRUCTURED_LOG_FIELDS:
+        default = False if field == "is-redacted" else None
+        ordered[field] = record.get(field, default)
+    return ordered
+
+
+def serialize_diagnostic_record(record: Dict[str, Any]) -> str:
+    return json.dumps(format_diagnostic_record(record), ensure_ascii=False)
+
+
+def format_console_record(record: Dict[str, Any], max_message_length: int = 250) -> str:
+    """Format a diagnostic record for human-readable TTY output with ANSI colors."""
+    ts = record.get("timestamp", "")
+    # Extract HH:MM:SS from ISO timestamp
+    time_part = ts[11:19] if len(ts) >= 19 else ts
+    level = (record.get("level") or "info").upper()
+    message = record.get("message", "")
+    if len(message) > max_message_length:
+        message = message[:max_message_length] + "…"
+
+    ids = []
+    for key in ("request-id", "conversation-id", "message-id"):
+        val = record.get(key)
+        if val:
+            ids.append(f"{key}={val}")
+    id_suffix = f" ({' | '.join(ids)})" if ids else ""
+
+    color = _ANSI_COLORS.get((record.get("level") or "info").lower(), "")
+    reset = _ANSI_RESET if color else ""
+    return f"{color}[{time_part}] {level} {message}{id_suffix}{reset}"
+
+
+def render_diagnostic(record: Dict[str, Any]) -> str:
+    """Return TTY-friendly or JSON output depending on whether stdout is a terminal."""
+    if sys.stdout.isatty():
+        return format_console_record(record)
+    return serialize_diagnostic_record(record)
+
+
+def emit_structured_log(
+    level: str,
+    message: str,
+    operation: str = Operation.EVALUATE,
+    *,
+    logger: logging.Logger,
+    diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    run_context: Optional[Dict[str, Any]] = None,
+) -> None:
+    """Emit a structured log entry.
+
+    Formats via format_structured_log_entry, optionally appends to
+    diagnostic_records, then logs via render_diagnostic (TTY-friendly or JSON).
+
+    Args:
+        level: One of "debug", "info", "warning", "error".
+        message: Human-readable log message.
+        operation: The CLI operation step (e.g. Operation.SEND_PROMPT).
+        logger: Logger to emit through.
+        diagnostic_records: If provided, the structured entry is appended here.
+        run_context: Full run context override (request-id, conversation-id,
+            message-id). Defaults to nulls with the given operation.
+    """
+    log_level_int = getattr(logging, level.upper(), logging.INFO)
+    if diagnostic_records is None and not logger.isEnabledFor(log_level_int):
+        return
+
+    context = run_context or {
+        "request-id": None,
+        "conversation-id": None,
+        "message-id": None,
+        "operation": operation,
+    }
+    entry = format_structured_log_entry(
+        level=level,
+        message=message,
+        logger_name=logger.name,
+        run_context=context,
+    )
+    if diagnostic_records is not None:
+        diagnostic_records.append(entry)
+    try:
+        logger.log(getattr(logging, level.upper(), logging.INFO), render_diagnostic(entry))
+    except Exception:
+        pass

package/src/clients/cli/cli_logging/logging_utils.py

@@ -0,0 +1,144 @@
+import logging
+import re
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class LogLevel(str, Enum):
+    """Log level enum. Inherits from str so comparisons like level == "debug" work."""
+    DEBUG = "debug"
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+class Operation(str, Enum):
+    """CLI operation steps for structured log entries."""
+    SETUP = "setup"
+    AUTHENTICATE = "authenticate"
+    VALIDATE_ENV = "validate-env"
+    LOAD_PROMPTS = "load-prompts"
+    FETCH_AGENTS = "fetch-agents"
+    SEND_PROMPT = "send-prompt"
+    EVALUATE = "evaluate"
+    WRITE_OUTPUT = "write-output"
+
+
+ALLOWED_LOG_LEVELS = tuple(level.value for level in LogLevel)
+LOG_LEVEL_MAP = {
+    LogLevel.DEBUG: logging.DEBUG,
+    LogLevel.INFO: logging.INFO,
+    LogLevel.WARNING: logging.WARNING,
+    LogLevel.ERROR: logging.ERROR,
+}
+
+STRUCTURED_LOG_FIELDS = (
+    "timestamp",
+    "level",
+    "operation",
+    "request-id",
+    "conversation-id",
+    "message-id",
+    "logger",
+    "message",
+    "is-redacted",
+)
+
+
+def normalize_log_level(value: Optional[str]) -> Optional[str]:
+    if value is None:
+        return None
+    return value.strip().lower()
+
+
+def resolve_log_level(
+    log_level_values: Optional[List[str]],
+) -> Tuple[Optional[str], Optional[str]]:
+    values = log_level_values or []
+    if not values:
+        return "info", None
+
+    # Use the last value provided (aligns with Node.js wrapper behavior).
+    last = normalize_log_level(values[-1])
+    if last not in ALLOWED_LOG_LEVELS:
+        return (
+            None,
+            "Invalid value for --log-level. Supported values are: "
+            "debug, info, warning, error.",
+        )
+
+    return last, None
+
+
+def utc_iso_timestamp() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def build_run_context(
+    operation: str = "evaluate",
+    request_id: Optional[str] = None,
+    conversation_id: Optional[str] = None,
+    message_id: Optional[str] = None,
+) -> Dict[str, Optional[str]]:
+    return {
+        "request-id": request_id,
+        "conversation-id": conversation_id,
+        "message-id": message_id,
+        "operation": operation,
+    }
+
+
+_SECRET_PATTERNS = [
+    re.compile(r"(?i)(api[_-]?key\s*[:=]\s*)([^\s,;]+)"),
+    re.compile(r"(?i)(token\s*[:=]\s*)([^\s,;]+)"),
+    re.compile(r"(?i)(authorization\s*[:=]\s*bearer\s+)([^\s,;]+)"),
+    re.compile(r"(?i)(password\s*[:=]\s*)([^\s,;]+)"),
+]
+
+
+def redact_sensitive_content(message: Optional[str]) -> Tuple[str, bool]:
+    if message is None:
+        return "", False
+
+    redacted = message
+    changed = False
+    for pattern in _SECRET_PATTERNS:
+        updated = pattern.sub(r"\1***REDACTED***", redacted)
+        if updated != redacted:
+            changed = True
+            redacted = updated
+
+    # Fallback: match strings 32+ chars containing mixed case and digits
+    # (likely a credential/token) that weren't already caught above.
+    if (
+        "***REDACTED***" not in redacted
+        and re.search(
+            r"(?=[A-Za-z0-9_\-]*[A-Z])(?=[A-Za-z0-9_\-]*[a-z])"
+            r"(?=[A-Za-z0-9_\-]*[0-9])[A-Za-z0-9_\-]{32,}",
+            redacted,
+        )
+    ):
+        return "[REDACTED]", True
+
+    return redacted, changed
+
+
+def format_structured_log_entry(
+    level: str,
+    message: str,
+    logger_name: str,
+    run_context: Dict[str, Optional[str]],
+) -> Dict[str, Any]:
+    safe_message, is_redacted = redact_sensitive_content(message)
+    return {
+        "level": normalize_log_level(level) or "info",
+        "message": safe_message,
+        "logger": logger_name,
+        "timestamp": utc_iso_timestamp(),
+        "request-id": run_context.get("request-id"),
+        "conversation-id": run_context.get("conversation-id"),
+        "message-id": run_context.get("message-id"),
+        "operation": run_context.get("operation"),
+        "is-redacted": is_redacted,
+    }

package/src/clients/cli/common.py

@@ -0,0 +1,62 @@
+"""Shared types and constants for the CLI."""
+
+import re
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+def pascal_case_to_title(eval_name: str) -> str:
+    """Convert PascalCase evaluator name to space-separated display name.
+
+    e.g., "ToolCallAccuracy" → "Tool Call Accuracy"
+    """
+    return re.sub(r'(?<=[a-z])(?=[A-Z])', ' ', eval_name)
+
+# Canonical evaluator name constants
+RELEVANCE = "Relevance"
+COHERENCE = "Coherence"
+GROUNDEDNESS = "Groundedness"
+TOOL_CALL_ACCURACY = "ToolCallAccuracy"
+CITATIONS = "Citations"
+EXACT_MATCH = "ExactMatch"
+PARTIAL_MATCH = "PartialMatch"
+
+# Prerequisite constants
+REQUIRES_AZURE_OPENAI = "azure_openai"
+REQUIRES_TOOL_DEFINITIONS = "tool_definitions"
+
+# Evaluation status constants
+# Outcome statuses (agent responded, evaluators ran):
+STATUS_PASS = "pass"  # All evaluators scored above threshold
+STATUS_FAIL = "fail"  # At least one evaluator scored below threshold
+# Error state (evaluation couldn't complete):
+STATUS_ERROR = "error"  # API call failed / response couldn't be obtained
+# Thread-level aggregate status (multi-turn only):
+STATUS_PARTIAL = "partial"  # Some turns passed, some did not
+# Fallback for missing status:
+STATUS_UNKNOWN = "unknown"
+
+# System defaults when no file-level or env-level defaults are configured
+SYSTEM_DEFAULT_EVALUATORS = [
+    RELEVANCE,
+    COHERENCE,
+]
+
+
+# Mapping from evaluator name to the key used in evaluator output dicts
+METRIC_IDS = {
+    RELEVANCE: "relevance",
+    COHERENCE: "coherence",
+    GROUNDEDNESS: "groundedness",
+    TOOL_CALL_ACCURACY: "tool_call_accuracy",
+    CITATIONS: "citations",
+    EXACT_MATCH: "exact_match",
+    PARTIAL_MATCH: "partial_match",
+}
+
+
+@dataclass
+class RegistryEntry:
+    type: str  # "llm", "tool", or "non-llm"
+    requires: List[str]
+    default_threshold: Optional[float]

package/src/clients/cli/custom_evaluators/CitationsEvaluator.py

@@ -17,8 +17,8 @@ from typing import Dict, Any, Optional
 class CitationFormat(Enum):
     """Enum for different citation formats supported by the evaluator."""
     OAI_UNICODE = "oai_unicode"  # New format: \ue200cite\ue202turn{X}search{Y}\ue201
-    LEGACY_BRACKET = "legacy_bracket"  # Old format: [^i^]
-    AUTO = "auto"  # Automatically detect both formats
+    LEGACY_BRACKET = "bracket"  # Old format: [^i^]
+    AUTO = "mixed"  # Automatically detect both formats
 
 
 class CitationsEvaluator:
@@ -141,7 +141,7 @@ class CitationsEvaluator:
 
         results = {
             "citation_format": self.citation_format.value,
-            "score": total_citations,
+            "citations": total_citations,
             "result": "pass" if total_citations > 0 else "fail",
             "threshold": 1,
             "reason": " ".join(reason_parts)

package/src/clients/cli/custom_evaluators/ExactMatchEvaluator.py

@@ -1,8 +1,6 @@
-from azure.ai.evaluation import evaluate
-
 class ExactMatchEvaluator:
-    def __init__(self):
-        pass
+    def __init__(self, case_sensitive=False):
+        self.case_sensitive = case_sensitive
 
     def __call__(self, *, response: str, expected_answer: str, **kwargs):
         if response is None or response.strip() == "":
@@ -11,15 +9,17 @@ class ExactMatchEvaluator:
         if expected_answer is None:
             raise ValueError("Expected answer cannot be None.")
 
-        # Case-sensitive exact match (mimics C# StringComparison.InvariantCulture)
-        is_match = response.strip() == expected_answer.strip()
+        resp = response.strip()
+        exp = expected_answer.strip()
+
+        if not self.case_sensitive:
+            resp = resp.lower()
+            exp = exp.lower()
+
+        is_match = resp == exp
 
         return {
             "exact_match": 1.0 if is_match else 0.0,
-            "exact_match_result": "pass" if is_match else "fail",
-            "exact_match_threshold": 1.0,
+            "result": "pass" if is_match else "fail",
             "exact_match_reason": "Exact match found" if is_match else "No exact match found"
         }
-
-
-exact_match_evaluator = ExactMatchEvaluator()

package/src/clients/cli/custom_evaluators/PartialMatchEvaluator.py

@@ -1,5 +1,3 @@
-from azure.ai.evaluation import evaluate
-
 class PartialMatchEvaluator:
     def __init__(self, case_sensitive=False):
         self.case_sensitive = case_sensitive
@@ -25,15 +23,7 @@ class PartialMatchEvaluator:
         else:
             score = 0.0
 
-        threshold = 0.5  # 50% match threshold
-        is_pass = score >= threshold
-
         return {
             "partial_match": score,
-            "partial_match_result": "pass" if is_pass else "fail",
-            "partial_match_threshold": threshold,
-            "partial_match_reason": f"Match score: {score:.3f} ({'above' if is_pass else 'below'} threshold {threshold})"
+            "partial_match_reason": f"Match score: {score:.3f}"
         }
-
-
-partial_match_evaluator = PartialMatchEvaluator(case_sensitive=False)

package/src/clients/cli/evaluator_resolver.py

@@ -0,0 +1,150 @@
+"""Evaluator resolution module for per-prompt evaluator configuration.
+
+Resolves which evaluators to run on each prompt by merging prompt-level config
+with file-level defaults and system defaults, following extend/replace modes.
+"""
+
+import difflib
+import logging
+from typing import Any, Dict, Optional, Tuple
+
+from common import (
+    RELEVANCE,
+    COHERENCE,
+    GROUNDEDNESS,
+    TOOL_CALL_ACCURACY,
+    CITATIONS,
+    EXACT_MATCH,
+    PARTIAL_MATCH,
+    REQUIRES_AZURE_OPENAI,
+    REQUIRES_TOOL_DEFINITIONS,
+    SYSTEM_DEFAULT_EVALUATORS,
+    RegistryEntry,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Static registry of available evaluators per data-model.md
+EVALUATOR_REGISTRY: Dict[str, RegistryEntry] = {
+    RELEVANCE: RegistryEntry(type="llm", requires=[REQUIRES_AZURE_OPENAI], default_threshold=3),
+    COHERENCE: RegistryEntry(type="llm", requires=[REQUIRES_AZURE_OPENAI], default_threshold=3),
+    GROUNDEDNESS: RegistryEntry(type="llm", requires=[REQUIRES_AZURE_OPENAI], default_threshold=3),
+    TOOL_CALL_ACCURACY: RegistryEntry(type="tool", requires=[REQUIRES_AZURE_OPENAI, REQUIRES_TOOL_DEFINITIONS], default_threshold=3),
+    CITATIONS: RegistryEntry(type="non-llm", requires=[], default_threshold=1),
+    EXACT_MATCH: RegistryEntry(type="non-llm", requires=[], default_threshold=None),
+    PARTIAL_MATCH: RegistryEntry(type="non-llm", requires=[], default_threshold=0.5),
+}
+
+
+def validate_evaluator_names(evaluator_map: Dict[str, Any]) -> None:
+    """Validate that all evaluator names in the map exist in the registry.
+
+    Raises ValueError with categorized valid names and
+    'Did you mean?' suggestions for close matches.
+    """
+    invalid_names = [name for name in evaluator_map if name not in EVALUATOR_REGISTRY]
+    if not invalid_names:
+        return
+
+    # Categorize valid evaluators for the error message
+    llm_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "llm"]
+    tool_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "tool"]
+    non_llm_evals = [n for n, r in EVALUATOR_REGISTRY.items() if r.type == "non-llm"]
+
+    lines = []
+    for name in invalid_names:
+        lines.append(f'Unknown evaluator "{name}".')
+        close = difflib.get_close_matches(name, EVALUATOR_REGISTRY.keys(), n=1, cutoff=0.5)
+        if close:
+            lines.append(f'Did you mean "{close[0]}"?')
+
+    lines.append("")
+    lines.append("Valid evaluators are:")
+    lines.append(f"  - {', '.join(llm_evals)} (LLM-based)")
+    lines.append(f"  - {', '.join(tool_evals)} (tool evaluation)")
+    lines.append(f"  - {', '.join(non_llm_evals)} (non-LLM)")
+
+    raise ValueError("\n".join(lines))
+
+
+def check_prerequisites(
+    evaluator_name: str,
+    available_context: Dict[str, bool],
+) -> Tuple[bool, Optional[str]]:
+    """Check if prerequisites for an evaluator are available.
+
+    Returns (True, None) if all prerequisites are met, or
+    (False, warning_message) if a prerequisite is missing.
+    """
+    registry_entry = EVALUATOR_REGISTRY.get(evaluator_name)
+    if not registry_entry:
+        return False, f"Unknown evaluator: {evaluator_name}"
+
+    for req in registry_entry.requires:
+        if not available_context.get(req, False):
+            msg = (
+                f"Skipping evaluator '{evaluator_name}': "
+                f"missing prerequisite '{req}'"
+            )
+            return False, msg
+
+    return True, None
+
+
+def resolve_default_evaluators(file_defaults: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """Resolve effective default evaluators, falling back to system defaults.
+
+    Precedence: file-level defaults > system defaults.
+    An explicit empty dict means "no default evaluators".
+    """
+    # File-level defaults (including explicit empty dict)
+    if file_defaults is not None:
+        return file_defaults
+
+    # System defaults
+    return {name: {} for name in SYSTEM_DEFAULT_EVALUATORS}
+
+
+def resolve_evaluators_for_prompt(
+    prompt_evaluators: Optional[Dict[str, Any]],
+    evaluators_mode: str,
+    prompt: str,
+    default_evaluators: Dict[str, Any],
+) -> Dict[str, Any]:
+    """Resolve which evaluators to run for a single prompt.
+
+    Args:
+        prompt_evaluators: Per-prompt evaluator config (None if not specified).
+        evaluators_mode: How to combine with defaults ("extend" or "replace").
+        prompt: The prompt text (used in warning messages).
+        default_evaluators: Resolved default evaluators (from resolve_default_evaluators).
+
+    Returns:
+        Resolved EvaluatorMap (dict of evaluator_name -> options).
+    """
+    # No prompt-level config → use defaults
+    if prompt_evaluators is None:
+        return dict(default_evaluators)
+
+    if evaluators_mode == "replace":
+        if not prompt_evaluators:
+            logger.warning(
+                "Empty evaluators with 'replace' mode for prompt: '%s'. "
+                "No evaluators will run.",
+                prompt[:80],
+            )
+        return dict(prompt_evaluators)
+
+    # mode == "extend": merge defaults with prompt overrides (prompt wins on conflict)
+    merged = dict(default_evaluators)
+    merged.update(prompt_evaluators)
+    return merged
+
+
+def get_evaluator_threshold(evaluator_name: str, options: Dict[str, Any]) -> Optional[float]:
+    """Get the threshold for an evaluator, with option override support."""
+    if "threshold" in options:
+        return options["threshold"]
+    entry = EVALUATOR_REGISTRY.get(evaluator_name)
+    return entry.default_threshold if entry else None