@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.4.0-preview.1

package/src/clients/cli/parallel_executor.py

@@ -0,0 +1,57 @@
+"""Parallel prompt execution utilities.
+
+This module provides a minimal reusable executor that preserves input order.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import Any, Callable, Generic, Iterable, List, Optional, TypeVar
+
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+@dataclass
+class WorkerResult(Generic[R]):
+    """Result model used to preserve input ordering and capture failures."""
+
+    index: int
+    value: Optional[R] = None
+    error: Optional[Exception] = None
+
+
+def execute_in_parallel(
+    items: Iterable[T],
+    worker: Callable[[T, int], R],
+    max_workers: int,
+) -> List[WorkerResult[R]]:
+    """Execute worker function in parallel while preserving input order.
+
+    `worker` receives `(item, index)` and returns a value.
+    """
+    indexed_items = list(enumerate(items))
+    if not indexed_items:
+        return []
+
+    normalized_workers = max(1, min(max_workers, len(indexed_items)))
+    results: List[WorkerResult[R]] = [WorkerResult(index=i) for i, _ in indexed_items]
+
+    with ThreadPoolExecutor(max_workers=normalized_workers) as executor:
+        future_map = {
+            executor.submit(worker, item, index): index
+            for index, item in indexed_items
+        }
+
+        for future in as_completed(future_map):
+            index = future_map[future]
+            try:
+                results[index] = WorkerResult(index=index, value=future.result())
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except Exception as exc:
+                results[index] = WorkerResult(index=index, error=exc)
+
+    return results

package/src/clients/cli/requirements.txt

@@ -6,7 +6,7 @@ msal[broker]>=1.34,<2
 msal-extensions>=1.3.1
 packaging>=20.0
 PyJWT>=2.11.0
-python-dotenv==1.1.1
+python-dotenv==1.2.2
 markdown==3.8.2
 promptflow>=1.18.1
 questionary>=2.1.1

package/src/clients/cli/response_extractor.py

@@ -573,25 +573,23 @@ class EnhancedResponseExtractor:
                 }
             }
 
-def extract_enhanced_responses(responses: List[Tuple[str, str]], log_level: str = "info") -> List[Dict[str, Any]]:
+def extract_enhanced_response(raw_response: str, log_level: str = "info") -> Dict[str, Any]:
     """
-    Extract enhanced response information for multiple responses.
-
+    Extract enhanced response information from a raw response string.
+    
     Args:
-        responses: List of (prompt_text, raw_response_string) tuples, one per prompt
-                   sent to the chat API. Order and duplicates are preserved.
-
+        raw_response: Raw response string from the agent
+        log_level: Logging level for the extraction process (default: "info")
+        
     Returns:
-        List of enhanced response dicts (one per prompt, same order as input).
+        A dictionary containing the enhanced response information, including:
+        - "response": Reconstructed message flow with tool calls and results
+        - "tool_definitions": List of tool definitions extracted from telemetry
+        - "raw_response_text": Original response text for backward compatibility
+        - "metadata": Additional metadata such as conversation ID, request ID, etc.
     """
     extractor = EnhancedResponseExtractor(log_level=log_level)
-    enhanced_responses = []
-
-    for prompt, raw_response in responses:
-        enhanced = extractor.extract_enhanced_response(raw_response)
-        enhanced_responses.append(enhanced)
-
-    return enhanced_responses
+    return extractor.extract_enhanced_response(raw_response)
 
 def get_response_text_for_evaluation(enhanced_response: Dict[str, Any]) -> str:
     """

package/src/clients/cli/retry_policy.py

@@ -0,0 +1,52 @@
+"""Retry utilities for transient HTTP failures in evaluation flows."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Optional
+
+RETRYABLE_HTTP_STATUS_CODES = {429, 503, 504}
+MAX_BACKOFF_SECONDS = 60
+
+
+def is_retryable_status(status_code: Optional[int]) -> bool:
+    """Return True for transient HTTP status codes covered by the spec."""
+    if status_code is None:
+        return False
+    return int(status_code) in RETRYABLE_HTTP_STATUS_CODES
+
+
+def get_backoff_seconds(attempt: int) -> int:
+    """Return exponential backoff delay capped at MAX_BACKOFF_SECONDS.
+
+    Examples: 2, 4, 8 for attempts 1..3.
+    """
+    if attempt < 1:
+        raise ValueError("attempt must be >= 1")
+    return min(2 ** attempt, MAX_BACKOFF_SECONDS)
+
+
+def get_retry_after_seconds(retry_after_header: Optional[str]) -> Optional[int]:
+    """Parse Retry-After header value (delay-seconds or HTTP-date per RFC 7231)."""
+    if retry_after_header is None:
+        return None
+
+    value = retry_after_header.strip()
+    if not value:
+        return None
+
+    # Try delay-seconds (integer) first
+    try:
+        return max(0, int(value))
+    except ValueError:
+        pass
+
+    # Try HTTP-date format (RFC 7231 §7.1.3)
+    try:
+        retry_date = parsedate_to_datetime(value)
+        now = datetime.now(timezone.utc)
+        delta = int((retry_date - now).total_seconds())
+        return max(0, delta)
+    except (ValueError, TypeError):
+        return None

package/src/clients/cli/samples/multiturn_example.json

@@ -0,0 +1,35 @@
+{
+  "schemaVersion": "1.2.0",
+  "default_evaluators": {
+    "Relevance": {},
+    "Coherence": {}
+  },
+  "items": [
+    {
+      "prompt": "What is Microsoft Graph?",
+      "expected_response": "Microsoft Graph is a gateway to data and intelligence in Microsoft 365."
+    },
+    {
+      "name": "Travel planning conversation",
+      "description": "Multi-turn thread testing context retention across turns",
+      "turns": [
+        {
+          "prompt": "I'm planning a trip to Seattle next week.",
+          "expected_response": "I can help you plan your Seattle trip."
+        },
+        {
+          "prompt": "What's the weather going to be like?",
+          "expected_response": "Seattle weather is typically mild with possible rain."
+        },
+        {
+          "prompt": "Should I bring a rain jacket?",
+          "expected_response": "Yes, it's always a good idea to bring rain gear to Seattle.",
+          "evaluators": {
+            "Groundedness": { "threshold": 4 }
+          },
+          "evaluators_mode": "extend"
+        }
+      ]
+    }
+  ]
+}

package/src/clients/cli/throttle_gate.py

@@ -0,0 +1,82 @@
+"""Per-API throttle gate support for transient HTTP 429 handling."""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class GateState:
+    """Snapshot state for diagnostics and tests."""
+
+    api_name: str
+    blocked_until_epoch: float
+    is_blocked: bool
+    last_retry_after_seconds: Optional[int]
+
+
+class ThrottleGate:
+    """Thread-safe per-API gate that pauses workers until the block window elapses."""
+
+    def __init__(self, api_name: str) -> None:
+        self.api_name = api_name
+        self._lock = threading.Lock()
+        self._blocked_until_epoch = 0.0
+        self._last_retry_after_seconds: Optional[int] = None
+
+    def apply_retry_after(self, retry_after_seconds: int) -> float:
+        """Apply retry-after duration and keep the maximum active block window.
+
+        Returns the current effective blocked-until epoch.
+        """
+        retry_after_seconds = max(0, int(retry_after_seconds))
+        candidate = time.time() + retry_after_seconds
+
+        with self._lock:
+            if candidate > self._blocked_until_epoch:
+                self._blocked_until_epoch = candidate
+            self._last_retry_after_seconds = retry_after_seconds
+            return self._blocked_until_epoch
+
+    MAX_GATE_WAIT_SECONDS = 300.0
+
+    def wait_if_blocked(self) -> float:
+        """Sleep until the gate opens. Returns the total slept duration in seconds.
+
+        Re-checks the block window after each sleep to handle concurrent
+        ``apply_retry_after`` calls that extend the window (avoids TOCTOU).
+        Raises ``TimeoutError`` if the total wait exceeds ``MAX_GATE_WAIT_SECONDS``.
+        """
+        total_slept = 0.0
+        while True:
+            with self._lock:
+                delay = max(0.0, self._blocked_until_epoch - time.time())
+            if delay <= 0:
+                return total_slept
+            if total_slept + delay > self.MAX_GATE_WAIT_SECONDS:
+                raise TimeoutError(
+                    f"ThrottleGate '{self.api_name}' exceeded maximum wait of "
+                    f"{self.MAX_GATE_WAIT_SECONDS}s (slept {total_slept:.1f}s so far)."
+                )
+            time.sleep(delay)
+            total_slept += delay
+
+    def clear(self) -> None:
+        """Reset the gate to unblocked state."""
+        with self._lock:
+            self._blocked_until_epoch = 0.0
+            self._last_retry_after_seconds = None
+
+    def state(self) -> GateState:
+        """Return immutable snapshot state."""
+        with self._lock:
+            now = time.time()
+            return GateState(
+                api_name=self.api_name,
+                blocked_until_epoch=self._blocked_until_epoch,
+                is_blocked=self._blocked_until_epoch > now,
+                last_retry_after_seconds=self._last_retry_after_seconds,
+            )

package/src/clients/node-js/bin/runevals.js

@@ -8,8 +8,10 @@ import { ensurePythonRuntime, getCacheDir } from '../lib/python-runtime.js';
 import { ensureVenv, executePythonCli } from '../lib/venv-manager.js';
 import { getCacheStats, clearCache, formatBytes } from '../lib/cache-utils.js';
 import { checkPackageExpiry } from '../lib/expiry-check.js';
+import { recordAcceptance, checkAcceptance } from '../lib/eula-manager.js';
 import { ProgressReporter } from '../lib/progress.js';
 import { _loadEnvFile as loadEnvFile, _loadUserEnvOverride } from '../lib/env-loader.js';
+import { normalizeAgentId } from '../lib/agent-id.js';
 
 // Check package expiry (exits if expired, warns if close to expiry)
 checkPackageExpiry();
@@ -22,20 +24,13 @@ const packageJsonPath = path.join(__dirname, '..', '..', '..', '..', 'package.js
 const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
 const VERSION = packageJson.version;
 
+const EULA_URL = 'https://aka.ms/evaltoolterms';
+
 // Path to Python CLI and requirements
 const PYTHON_CLI_DIR = path.join(__dirname, '..', '..', 'cli');
 const MAIN_SCRIPT = path.join(PYTHON_CLI_DIR, 'main.py');
 const REQUIREMENTS_FILE = path.join(PYTHON_CLI_DIR, 'requirements.txt');
 
-/**
- * Display usage terms notice
- * Called before running evaluations (but not for --init-only, cache commands, or --signout).
- * This notice is always displayed per legal requirements (FR-006).
- */
-function displayUsageTerms() {
-  console.log('By using this tool, you agree to the Terms of Use: https://aka.ms/evaltoolterms\n');
-}
-
 /**
  * Set default environment constants that cannot be overridden
  * This ensures these values are always set regardless of .env files
@@ -176,21 +171,80 @@ async function main() {
     .option('--prompts-file <file>', 'JSON file with prompts and expected responses')
     .option('-o, --output <file>', 'output file (JSON, CSV, or HTML)')
     .option('-i, --interactive', 'interactive mode (enter prompts interactively)')
+    .option('--concurrency <number>', 'max prompts to process in parallel (1-5)')
     .option('--m365-agent-id <id>', 'agent ID (overrides env vars and auto-construction)')
     .option('--env <environment>', 'environment name (loads env/.env.<environment>)', 'local')
     .option('--init-only', 'only initialize Python environment, don\'t run evaluations')
     .option('--cache-info', 'show cache information and statistics')
     .option('--cache-clear', 'clear the cache (removes Python runtime and venv)')
     .option('--cache-dir', 'print the cache directory path')
-    .option('--signout', 'sign out and clear cached authentication tokens');
+    .option('--signout', 'sign out and clear cached authentication tokens')
+    .action(() => {
+      // Default command — handled by the main flow below parseAsync()
+    });
+
+  program
+    .command('accept-eula')
+    .description('Accept the End User License Agreement (EULA)')
+    .action(async () => {
+      const config = (await import('../config/default.js')).default;
+      try {
+        await recordAcceptance(config.eula.version);
+        console.log('EULA has been accepted');
+        process.exit(0);
+      } catch (err) {
+        console.error(
+          `⚠️  Unable to persist EULA acceptance: ${err.message}`,
+        );
+        console.error(
+          'Please ensure the directory ~/.m365-copilot-agent-evals/ is writable.',
+        );
+        process.exit(1);
+      }
+    });
 
-  program.parse(process.argv);
+  await program.parseAsync(process.argv);
   const options = program.opts();
   const effectiveLogLevel = resolveLogLevel(options);
   const outputMode = deriveWrapperOutputMode(effectiveLogLevel);
   const wrapperVerbose = outputMode.verbose;
   const wrapperQuiet = outputMode.quiet;
 
+  // === EULA Enforcement Gate ===
+  // Block all commands until EULA is accepted (FR-010, FR-011).
+  // accept-eula subcommand, --help, and --version are already handled
+  // by Commander during program.parse() and exit before reaching here.
+  const config = (await import('../config/default.js')).default;
+  const { accepted, stale } = await checkAcceptance(config.eula.version);
+  if (!accepted) {
+    if (stale) {
+      console.error(
+`==============================================================
+The End User License Agreement (EULA) has been updated.
+Please review the updated terms at:
+${EULA_URL}
+
+To accept the updated EULA, please execute the following command:
+
+    runevals accept-eula
+
+==============================================================`);
+    } else {
+      console.error(
+`==============================================================
+In order to use this tool you must accept the End User License
+Agreement (EULA) found at:
+${EULA_URL}
+
+To accept the EULA, please execute the following command:
+
+    runevals accept-eula
+
+==============================================================`);
+    }
+    process.exit(2);
+  }
+
   // Handle cache commands first (they don't need environment validation or config)
   if (options.cacheInfo) {
     console.log('🗂️  Cache Information\n');
@@ -251,8 +305,7 @@ async function main() {
 
   // === From here on, we're running actual evals - load config and env files ===
 
-  displayUsageTerms();
-  // Load build-time config
+  // Load build-time config (already loaded above for EULA check)
   await setDefaultEnvironmentConstants();
 
   // Load environment files
@@ -322,11 +375,17 @@ async function main() {
     }
   }
 
-  // Resolve agent ID from environment if not explicitly provided via CLI flag
-  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID
+  // Resolve agent ID from environment if not explicitly provided via CLI flag.
+  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID.
+  // Then normalize via shared helper and sync to process.env so downstream
+  // readers (and the python CLI) see the canonical form.
   if (!resolvedAgentId) {
     resolvedAgentId = envVars['M365_AGENT_ID'] || process.env.M365_AGENT_ID;
-    if (resolvedAgentId && !wrapperQuiet) {
+  }
+  resolvedAgentId = normalizeAgentId(resolvedAgentId);
+  if (resolvedAgentId) {
+    process.env.M365_AGENT_ID = resolvedAgentId;
+    if (!options.m365AgentId && !wrapperQuiet) {
       console.log(`🤖 Agent ID: ${resolvedAgentId}`);
     }
   }
@@ -458,6 +517,10 @@ async function main() {
   if (options.prompts && options.prompts.length > 0) {
     pythonArgs.push('--prompts', ...options.prompts);
   }
+
+  if (options.concurrency !== undefined) {
+    pythonArgs.push('--concurrency', String(options.concurrency));
+  }
   
   if (options.expected && options.expected.length > 0) {
     pythonArgs.push('--expected', ...options.expected);

package/src/clients/node-js/config/default.js

@@ -2,7 +2,7 @@
  * Build-time injected default values
  * DO NOT EDIT - This file is auto-generated during build.
  *
- * Generated: 2026-04-01T19:33:48.937Z
+ * Generated: 2026-04-22T20:44:41.713Z
  *
  * @copyright Microsoft Corporation. All rights reserved.
  * @license MIT
@@ -21,5 +21,9 @@ export default {
 
     /** Scenario Header for Copilot API */
     scenarioHeader: "agenticevaluation"
+  },
+  eula: {
+    /** EULA version string for acceptance tracking */
+    version: "2026-04-01"
   }
 };

package/src/clients/node-js/lib/agent-id.js

@@ -0,0 +1,12 @@
+/**
+ * Normalize an M365 agent ID by appending '.declarativeAgent' when the value
+ * has no '.' segment. Returns the input unchanged when null/undefined/empty
+ * or when it already contains a dot.
+ *
+ * @param {string|null|undefined} id - The raw agent ID value.
+ * @returns {string|null|undefined} The normalized agent ID.
+ */
+export function normalizeAgentId(id) {
+  if (!id) return id;
+  return id.includes('.') ? id : `${id}.declarativeAgent`;
+}

package/src/clients/node-js/lib/env-loader.js

@@ -3,6 +3,7 @@
  * Handles .env.local, .env.local.user, and other env file formats.
  */
 
+import { parse as dotenvParse } from 'dotenv';
 import fs from 'fs';
 import path from 'path';
 
@@ -21,7 +22,8 @@ const AGENT_ID_ALIASES = [
 
 /**
  * Load environment variables from a .env-style file.
- * Skips blank lines and comments. Protected keys are ignored with a warning.
+ * Uses dotenv.parse() for standards-compliant parsing (handles quoted values,
+ * inline comments, escape sequences). Protected keys are ignored with a warning.
  * Malformed lines (no '=' separator) are skipped with a warning.
  * @param {string} envFilePath - Absolute path to the env file
  * @returns {Object|null} Parsed key-value pairs, or null if file cannot be read
@@ -34,32 +36,25 @@ export function _loadEnvFile(envFilePath) {
   const envVars = {};
   try {
     const content = fs.readFileSync(envFilePath, 'utf-8');
-    const lines = content.split('\n');
 
-    for (const line of lines) {
+    // Pre-scan for malformed lines (no '=') and emit warnings
+    for (const line of content.split('\n')) {
       const trimmedLine = line.trim();
       if (!trimmedLine || trimmedLine.startsWith('#')) {
         continue;
       }
-
-      const eqIndex = trimmedLine.indexOf('=');
-      if (eqIndex === -1) {
+      if (trimmedLine.indexOf('=') === -1) {
         console.warn(
           `⚠️ Ignoring malformed line in env file (missing '='): ${trimmedLine}`
         );
-        continue;
       }
+    }
 
-      const keyName = trimmedLine.slice(0, eqIndex).trim();
-      const value = trimmedLine
-        .slice(eqIndex + 1)
-        .trim()
-        .replace(/^(['"])(.*)\1$/, '$2');
-
-      if (!keyName) {
-        continue;
-      }
+    // Use dotenv.parse() for standards-compliant .env parsing
+    // (handles quoted values, inline comments, escape sequences, export prefix)
+    const parsed = dotenvParse(content);
 
+    for (const [keyName, value] of Object.entries(parsed)) {
       if (PROTECTED_KEYS.includes(keyName)) {
         console.warn(
           `⚠️ Ignoring ${keyName} from .env file (using built-in value)`

package/src/clients/node-js/lib/eula-manager.js

@@ -0,0 +1,78 @@
+/**
+ * EULA acceptance manager
+ *
+ * Manages reading and writing the EULA acceptance marker file at
+ * ~/.m365-copilot-agent-evals/eula-acceptance.json.
+ * This location is independent of the cache directory so acceptance
+ * survives --cache-clear operations.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+
+const EULA_DIR_NAME = '.m365-copilot-agent-evals';
+const EULA_FILE_NAME = 'eula-acceptance.json';
+
+/**
+ * Returns the EULA directory path (~/.m365-copilot-agent-evals/).
+ * @returns {string}
+ */
+export function getEulaDir() {
+  return path.join(os.homedir(), EULA_DIR_NAME);
+}
+
+/**
+ * Returns the full path to the acceptance marker file.
+ * @returns {string}
+ */
+export function getEulaFilePath() {
+  return path.join(getEulaDir(), EULA_FILE_NAME);
+}
+
+/**
+ * Write an acceptance marker for the given EULA version.
+ * Creates the directory if it doesn't exist.
+ * @param {string} version - EULA version string
+ * @returns {Promise<void>}
+ */
+export async function recordAcceptance(version) {
+  const dir = getEulaDir();
+  await fs.mkdir(dir, { recursive: true });
+  const marker = { version, acceptedAt: new Date().toISOString() };
+  await fs.writeFile(
+    getEulaFilePath(),
+    JSON.stringify(marker, null, 2),
+    'utf-8'
+  );
+}
+
+/**
+ * Check whether the EULA has been accepted for the required version.
+ * @param {string} requiredVersion - The version to check against
+ * @returns {Promise<{accepted: boolean, stale: boolean, marker: object|null}>}
+ */
+export async function checkAcceptance(requiredVersion) {
+  const marker = await _readMarker();
+  if (!marker) return { accepted: false, stale: false, marker: null };
+  if (marker.version !== requiredVersion)
+    return { accepted: false, stale: true, marker };
+  return { accepted: true, stale: false, marker };
+}
+
+/**
+ * Read and parse the acceptance marker file.
+ * Returns null if the file is missing, unreadable, or malformed.
+ * Exported with _ prefix for unit testing.
+ * @returns {Promise<object|null>}
+ */
+export async function _readMarker() {
+  try {
+    const raw = await fs.readFile(getEulaFilePath(), 'utf-8');
+    const parsed = JSON.parse(raw);
+    if (!parsed.version || !parsed.acceptedAt) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}

package/src/clients/node-js/lib/progress.js

@@ -605,18 +605,20 @@ export class ProgressReporter {
 
     this.phaseStatuses.set(phaseId, 'failed');
 
-    // Clear current line and display error
-    if (this.isInteractive) {
-      readline.clearLine(process.stdout, 0);
-      readline.cursorTo(process.stdout, 0);
-    }
+    if (!this.options.quiet) {
+      // Clear current line and display error
+      if (this.isInteractive) {
+        readline.clearLine(process.stdout, 0);
+        readline.cursorTo(process.stdout, 0);
+      }
 
-    console.log(`\n❌ Failed: ${phase.name}`);
-    console.log(`\nError: ${error.message}`);
-    console.log(`\nSuggested actions:`);
-    console.log(`  • Check your internet connection`);
-    console.log(`  • If behind a proxy, set HTTP_PROXY/HTTPS_PROXY`);
-    console.log(`  • Run with --verbose for detailed output`);
+      console.error(`\n❌ Failed: ${phase.name}`);
+      console.error(`\nError: ${error.message}`);
+      console.error(`\nSuggested actions:`);
+      console.error(`  • Check your internet connection`);
+      console.error(`  • If behind a proxy, set HTTP_PROXY/HTTPS_PROXY`);
+      console.error(`  • Run with --verbose for detailed output`);
+    }
 
     this.currentPhase = null;
     this.phaseStartTime = null;