@microsoft/m365-copilot-eval 1.4.0-preview.1 → 1.6.0-preview.1

package/src/clients/cli/common.py

@@ -4,6 +4,22 @@ import re
 from dataclasses import dataclass
 from typing import List, Optional
 
+MAX_CONCURRENCY = 5
+MAX_ATTEMPTS = 4  # Initial attempt + 3 retries
+MAX_TURNS_PER_THREAD = 20
+LONG_THREAD_WARNING_THRESHOLD = 10
+DEFAULT_PASS_THRESHOLD = 3
+
+# ── Environment variable name constants ──────────────────────────────
+ENV_AZURE_AI_OPENAI_ENDPOINT = "AZURE_AI_OPENAI_ENDPOINT"
+ENV_AZURE_AI_API_KEY = "AZURE_AI_API_KEY"
+ENV_AZURE_AI_API_VERSION = "AZURE_AI_API_VERSION"
+ENV_AZURE_AI_MODEL_NAME = "AZURE_AI_MODEL_NAME"
+ENV_TENANT_ID = "TENANT_ID"
+ENV_WORK_IQ_A2A_ENDPOINT = "WORK_IQ_A2A_ENDPOINT"
+ENV_WORK_IQ_A2A_CLIENT_ID = "WORK_IQ_A2A_CLIENT_ID"
+ENV_WORK_IQ_A2A_SCOPES = "WORK_IQ_A2A_SCOPES"
+
 
 def pascal_case_to_title(eval_name: str) -> str:
     """Convert PascalCase evaluator name to space-separated display name.
@@ -12,10 +28,12 @@ def pascal_case_to_title(eval_name: str) -> str:
     """
     return re.sub(r'(?<=[a-z])(?=[A-Z])', ' ', eval_name)
 
+
 # Canonical evaluator name constants
 RELEVANCE = "Relevance"
 COHERENCE = "Coherence"
 GROUNDEDNESS = "Groundedness"
+SIMILARITY = "Similarity"
 TOOL_CALL_ACCURACY = "ToolCallAccuracy"
 CITATIONS = "Citations"
 EXACT_MATCH = "ExactMatch"
@@ -48,6 +66,7 @@ METRIC_IDS = {
     RELEVANCE: "relevance",
     COHERENCE: "coherence",
     GROUNDEDNESS: "groundedness",
+    SIMILARITY: "similarity",
     TOOL_CALL_ACCURACY: "tool_call_accuracy",
     CITATIONS: "citations",
     EXACT_MATCH: "exact_match",
@@ -60,3 +79,37 @@ class RegistryEntry:
     type: str  # "llm", "tool", or "non-llm"
     requires: List[str]
     default_threshold: Optional[float]
+
+
+@dataclass(frozen=True)
+class RunConfig:
+    """Typed, immutable runtime configuration passed across module boundaries.
+
+    Use ``RunConfig.from_namespace(args)`` to build from argparse output.
+    Use ``dataclasses.replace(config, field=value)`` to derive new configs.
+    """
+    prompts: Optional[List[str]] = None
+    expected: Optional[List[str]] = None
+    prompts_file: Optional[str] = None
+    interactive: bool = False
+    m365_agent_id: Optional[str] = None
+    output: Optional[str] = None
+    log_level: Optional[List[str]] = None
+    effective_log_level: str = "info"
+    signout: bool = False
+    concurrency: int = MAX_CONCURRENCY
+
+    @classmethod
+    def from_namespace(cls, args) -> "RunConfig":
+        """Build a RunConfig from an argparse.Namespace."""
+        return cls(
+            prompts=args.prompts,
+            expected=args.expected,
+            prompts_file=args.prompts_file,
+            interactive=args.interactive,
+            m365_agent_id=args.m365_agent_id,
+            output=args.output,
+            log_level=args.log_level,
+            signout=args.signout,
+            concurrency=args.concurrency,
+        )

package/src/clients/cli/env_validator.py

@@ -0,0 +1,73 @@
+"""Environment validation and URL security checks."""
+
+import os
+import sys
+import urllib.parse
+from typing import List
+
+from cli_logging.cli_logger import emit_structured_log
+from cli_logging.logging_utils import Operation
+from common import (
+    ENV_AZURE_AI_OPENAI_ENDPOINT,
+    ENV_AZURE_AI_API_KEY,
+    ENV_AZURE_AI_API_VERSION,
+    ENV_AZURE_AI_MODEL_NAME,
+    ENV_WORK_IQ_A2A_ENDPOINT,
+    ENV_WORK_IQ_A2A_CLIENT_ID,
+    ENV_WORK_IQ_A2A_SCOPES,
+    ENV_TENANT_ID,
+)
+
+
+# Allowed endpoints for URL validation
+ALLOWED_ENDPOINTS = [
+    'substrate.office.com',
+    'graph.microsoft.com',
+]
+
+
+def validate_environment() -> None:
+    """Validate required environment variables."""
+    required_env_vars = [
+        ENV_AZURE_AI_OPENAI_ENDPOINT,
+        ENV_AZURE_AI_API_KEY,
+        ENV_AZURE_AI_API_VERSION,
+        ENV_AZURE_AI_MODEL_NAME,
+        ENV_WORK_IQ_A2A_ENDPOINT,
+        ENV_WORK_IQ_A2A_CLIENT_ID,
+        ENV_WORK_IQ_A2A_SCOPES,
+        ENV_TENANT_ID,
+    ]
+
+    missing_vars = [
+        var for var in required_env_vars if not os.environ.get(var)
+    ]
+    if missing_vars:
+        emit_structured_log(
+            "error",
+            "Missing required environment variables: "
+            f"{', '.join(missing_vars)}. Please ensure your .env file"
+            " contains all required configuration.",
+            operation=Operation.VALIDATE_ENV,
+        )
+        sys.exit(1)
+
+
+def validate_endpoint_url(url: str, allowed_domains: List[str]) -> None:
+    """Validate URL against security requirements."""
+    try:
+        parsed = urllib.parse.urlparse(url)
+    except Exception as e:
+        raise ValueError(f"Invalid URL format: {url}") from e
+
+    if parsed.scheme in ['javascript', 'data']:
+        raise ValueError(f"Dangerous URL scheme detected: {parsed.scheme}")
+
+    if parsed.scheme != 'https':
+        raise ValueError(f"Only HTTPS URLs are allowed, got: {parsed.scheme}")
+
+    if parsed.netloc not in allowed_domains:
+        raise ValueError(f"Domain not in allowed list: {parsed.netloc}")
+
+    if parsed.fragment:
+        raise ValueError("Fragment URLs are not allowed")