@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.5.0-preview.1

package/src/clients/cli/cli_logging/cli_logger.py

@@ -0,0 +1,33 @@
+"""Shared CLI logger instance and structured-log convenience wrapper.
+
+Every module in the CLI layer that needs to emit diagnostics imports from here
+instead of main.py, which avoids circular-import issues.
+"""
+
+import logging
+import sys
+from typing import Any, Dict, List
+
+from cli_logging.console_diagnostics import emit_structured_log as _emit_structured_log
+from cli_logging.logging_utils import LOG_LEVEL_MAP, Operation
+
+CLI_LOGGER_NAME = "m365.eval.cli"
+CLI_LOGGER = logging.getLogger(CLI_LOGGER_NAME)
+DIAGNOSTIC_RECORDS: List[Dict[str, Any]] = []
+
+
+def configure_cli_logging(effective_log_level: str) -> None:
+    if not CLI_LOGGER.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        CLI_LOGGER.addHandler(handler)
+        CLI_LOGGER.propagate = False
+    CLI_LOGGER.setLevel(LOG_LEVEL_MAP[effective_log_level])
+
+
+def emit_structured_log(level: str, message: str, operation: str = Operation.EVALUATE) -> None:
+    _emit_structured_log(
+        level, message, operation,
+        logger=CLI_LOGGER,
+        diagnostic_records=DIAGNOSTIC_RECORDS,
+    )

package/src/clients/cli/cli_logging/console_diagnostics.py

@@ -1,9 +1,15 @@
 import json
+import logging
 import sys
 from collections import OrderedDict
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
 
-from cli_logging.logging_utils import STRUCTURED_LOG_FIELDS
+from cli_logging.logging_utils import (
+    STRUCTURED_LOG_FIELDS,
+    Operation,
+    format_structured_log_entry,
+    redact_sensitive_content,
+)
 
 _ANSI_COLORS = {
     "debug": "\033[2m",     # dim
@@ -53,3 +59,51 @@ def render_diagnostic(record: Dict[str, Any]) -> str:
     if sys.stdout.isatty():
         return format_console_record(record)
     return serialize_diagnostic_record(record)
+
+
+def emit_structured_log(
+    level: str,
+    message: str,
+    operation: str = Operation.EVALUATE,
+    *,
+    logger: logging.Logger,
+    diagnostic_records: Optional[List[Dict[str, Any]]] = None,
+    run_context: Optional[Dict[str, Any]] = None,
+) -> None:
+    """Emit a structured log entry.
+
+    Formats via format_structured_log_entry, optionally appends to
+    diagnostic_records, then logs via render_diagnostic (TTY-friendly or JSON).
+
+    Args:
+        level: One of "debug", "info", "warning", "error".
+        message: Human-readable log message.
+        operation: The CLI operation step (e.g. Operation.SEND_PROMPT).
+        logger: Logger to emit through.
+        diagnostic_records: If provided, the structured entry is appended here.
+        run_context: Full run context override (request-id, conversation-id,
+            message-id). Defaults to nulls with the given operation.
+    """
+    log_level_int = getattr(logging, level.upper(), logging.INFO)
+    if diagnostic_records is None and not logger.isEnabledFor(log_level_int):
+        return
+
+    context = run_context or {
+        "request-id": None,
+        "conversation-id": None,
+        "message-id": None,
+        "operation": operation,
+    }
+    entry = format_structured_log_entry(
+        level=level,
+        message=message,
+        logger_name=logger.name,
+        run_context=context,
+    )
+    if diagnostic_records is not None:
+        diagnostic_records.append(entry)
+    try:
+        rendered, _ = redact_sensitive_content(render_diagnostic(entry))
+        logger.log(getattr(logging, level.upper(), logging.INFO), rendered)
+    except Exception:
+        pass

package/src/clients/cli/cli_logging/logging_utils.py

@@ -52,7 +52,6 @@ def normalize_log_level(value: Optional[str]) -> Optional[str]:
     return value.strip().lower()
 
 
-
 def resolve_log_level(
     log_level_values: Optional[List[str]],
 ) -> Tuple[Optional[str], Optional[str]]:

package/src/clients/cli/common.py

@@ -4,6 +4,22 @@ import re
 from dataclasses import dataclass
 from typing import List, Optional
 
+MAX_CONCURRENCY = 5
+MAX_ATTEMPTS = 4  # Initial attempt + 3 retries
+MAX_TURNS_PER_THREAD = 20
+LONG_THREAD_WARNING_THRESHOLD = 10
+DEFAULT_PASS_THRESHOLD = 3
+
+# ── Environment variable name constants ──────────────────────────────
+ENV_AZURE_AI_OPENAI_ENDPOINT = "AZURE_AI_OPENAI_ENDPOINT"
+ENV_AZURE_AI_API_KEY = "AZURE_AI_API_KEY"
+ENV_AZURE_AI_API_VERSION = "AZURE_AI_API_VERSION"
+ENV_AZURE_AI_MODEL_NAME = "AZURE_AI_MODEL_NAME"
+ENV_TENANT_ID = "TENANT_ID"
+ENV_WORK_IQ_A2A_ENDPOINT = "WORK_IQ_A2A_ENDPOINT"
+ENV_WORK_IQ_A2A_CLIENT_ID = "WORK_IQ_A2A_CLIENT_ID"
+ENV_WORK_IQ_A2A_SCOPES = "WORK_IQ_A2A_SCOPES"
+
 
 def pascal_case_to_title(eval_name: str) -> str:
     """Convert PascalCase evaluator name to space-separated display name.
@@ -12,10 +28,12 @@ def pascal_case_to_title(eval_name: str) -> str:
     """
     return re.sub(r'(?<=[a-z])(?=[A-Z])', ' ', eval_name)
 
+
 # Canonical evaluator name constants
 RELEVANCE = "Relevance"
 COHERENCE = "Coherence"
 GROUNDEDNESS = "Groundedness"
+SIMILARITY = "Similarity"
 TOOL_CALL_ACCURACY = "ToolCallAccuracy"
 CITATIONS = "Citations"
 EXACT_MATCH = "ExactMatch"
@@ -25,6 +43,17 @@ PARTIAL_MATCH = "PartialMatch"
 REQUIRES_AZURE_OPENAI = "azure_openai"
 REQUIRES_TOOL_DEFINITIONS = "tool_definitions"
 
+# Evaluation status constants
+# Outcome statuses (agent responded, evaluators ran):
+STATUS_PASS = "pass"  # All evaluators scored above threshold
+STATUS_FAIL = "fail"  # At least one evaluator scored below threshold
+# Error state (evaluation couldn't complete):
+STATUS_ERROR = "error"  # API call failed / response couldn't be obtained
+# Thread-level aggregate status (multi-turn only):
+STATUS_PARTIAL = "partial"  # Some turns passed, some did not
+# Fallback for missing status:
+STATUS_UNKNOWN = "unknown"
+
 # System defaults when no file-level or env-level defaults are configured
 SYSTEM_DEFAULT_EVALUATORS = [
     RELEVANCE,
@@ -37,6 +66,7 @@ METRIC_IDS = {
     RELEVANCE: "relevance",
     COHERENCE: "coherence",
     GROUNDEDNESS: "groundedness",
+    SIMILARITY: "similarity",
     TOOL_CALL_ACCURACY: "tool_call_accuracy",
     CITATIONS: "citations",
     EXACT_MATCH: "exact_match",
@@ -49,3 +79,37 @@ class RegistryEntry:
     type: str  # "llm", "tool", or "non-llm"
     requires: List[str]
     default_threshold: Optional[float]
+
+
+@dataclass(frozen=True)
+class RunConfig:
+    """Typed, immutable runtime configuration passed across module boundaries.
+
+    Use ``RunConfig.from_namespace(args)`` to build from argparse output.
+    Use ``dataclasses.replace(config, field=value)`` to derive new configs.
+    """
+    prompts: Optional[List[str]] = None
+    expected: Optional[List[str]] = None
+    prompts_file: Optional[str] = None
+    interactive: bool = False
+    m365_agent_id: Optional[str] = None
+    output: Optional[str] = None
+    log_level: Optional[List[str]] = None
+    effective_log_level: str = "info"
+    signout: bool = False
+    concurrency: int = MAX_CONCURRENCY
+
+    @classmethod
+    def from_namespace(cls, args) -> "RunConfig":
+        """Build a RunConfig from an argparse.Namespace."""
+        return cls(
+            prompts=args.prompts,
+            expected=args.expected,
+            prompts_file=args.prompts_file,
+            interactive=args.interactive,
+            m365_agent_id=args.m365_agent_id,
+            output=args.output,
+            log_level=args.log_level,
+            signout=args.signout,
+            concurrency=args.concurrency,
+        )

package/src/clients/cli/env_validator.py

@@ -0,0 +1,73 @@
+"""Environment validation and URL security checks."""
+
+import os
+import sys
+import urllib.parse
+from typing import List
+
+from cli_logging.cli_logger import emit_structured_log
+from cli_logging.logging_utils import Operation
+from common import (
+    ENV_AZURE_AI_OPENAI_ENDPOINT,
+    ENV_AZURE_AI_API_KEY,
+    ENV_AZURE_AI_API_VERSION,
+    ENV_AZURE_AI_MODEL_NAME,
+    ENV_WORK_IQ_A2A_ENDPOINT,
+    ENV_WORK_IQ_A2A_CLIENT_ID,
+    ENV_WORK_IQ_A2A_SCOPES,
+    ENV_TENANT_ID,
+)
+
+
+# Allowed endpoints for URL validation
+ALLOWED_ENDPOINTS = [
+    'substrate.office.com',
+    'graph.microsoft.com',
+]
+
+
+def validate_environment() -> None:
+    """Validate required environment variables."""
+    required_env_vars = [
+        ENV_AZURE_AI_OPENAI_ENDPOINT,
+        ENV_AZURE_AI_API_KEY,
+        ENV_AZURE_AI_API_VERSION,
+        ENV_AZURE_AI_MODEL_NAME,
+        ENV_WORK_IQ_A2A_ENDPOINT,
+        ENV_WORK_IQ_A2A_CLIENT_ID,
+        ENV_WORK_IQ_A2A_SCOPES,
+        ENV_TENANT_ID,
+    ]
+
+    missing_vars = [
+        var for var in required_env_vars if not os.environ.get(var)
+    ]
+    if missing_vars:
+        emit_structured_log(
+            "error",
+            "Missing required environment variables: "
+            f"{', '.join(missing_vars)}. Please ensure your .env file"
+            " contains all required configuration.",
+            operation=Operation.VALIDATE_ENV,
+        )
+        sys.exit(1)
+
+
+def validate_endpoint_url(url: str, allowed_domains: List[str]) -> None:
+    """Validate URL against security requirements."""
+    try:
+        parsed = urllib.parse.urlparse(url)
+    except Exception as e:
+        raise ValueError(f"Invalid URL format: {url}") from e
+
+    if parsed.scheme in ['javascript', 'data']:
+        raise ValueError(f"Dangerous URL scheme detected: {parsed.scheme}")
+
+    if parsed.scheme != 'https':
+        raise ValueError(f"Only HTTPS URLs are allowed, got: {parsed.scheme}")
+
+    if parsed.netloc not in allowed_domains:
+        raise ValueError(f"Domain not in allowed list: {parsed.netloc}")
+
+    if parsed.fragment:
+        raise ValueError("Fragment URLs are not allowed")