@microsoft/m365-copilot-eval 1.3.0-preview.1 → 1.5.0-preview.1

package/src/clients/node-js/bin/runevals.js

@@ -8,8 +8,10 @@ import { ensurePythonRuntime, getCacheDir } from '../lib/python-runtime.js';
 import { ensureVenv, executePythonCli } from '../lib/venv-manager.js';
 import { getCacheStats, clearCache, formatBytes } from '../lib/cache-utils.js';
 import { checkPackageExpiry } from '../lib/expiry-check.js';
+import { recordAcceptance, checkAcceptance } from '../lib/eula-manager.js';
 import { ProgressReporter } from '../lib/progress.js';
 import { _loadEnvFile as loadEnvFile, _loadUserEnvOverride } from '../lib/env-loader.js';
+import { normalizeAgentId } from '../lib/agent-id.js';
 
 // Check package expiry (exits if expired, warns if close to expiry)
 checkPackageExpiry();
@@ -22,20 +24,13 @@ const packageJsonPath = path.join(__dirname, '..', '..', '..', '..', 'package.js
 const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
 const VERSION = packageJson.version;
 
+const EULA_URL = 'https://aka.ms/evaltoolterms';
+
 // Path to Python CLI and requirements
 const PYTHON_CLI_DIR = path.join(__dirname, '..', '..', 'cli');
 const MAIN_SCRIPT = path.join(PYTHON_CLI_DIR, 'main.py');
 const REQUIREMENTS_FILE = path.join(PYTHON_CLI_DIR, 'requirements.txt');
 
-/**
- * Display usage terms notice
- * Called before running evaluations (but not for --init-only, cache commands, or --signout).
- * This notice is always displayed per legal requirements (FR-006).
- */
-function displayUsageTerms() {
-  console.log('By using this tool, you agree to the Terms of Use: https://aka.ms/evaltoolterms\n');
-}
-
 /**
  * Set default environment constants that cannot be overridden
  * This ensures these values are always set regardless of .env files
@@ -43,10 +38,9 @@ function displayUsageTerms() {
  */
 async function setDefaultEnvironmentConstants() {
   const config = (await import('../config/default.js')).default;
-  process.env.M365_EVAL_CLIENT_ID = config.copilotApi.m365EvalClientId;
-  process.env.COPILOT_API_ENDPOINT = config.copilotApi.copilotApiEndpoint;
-  process.env.COPILOT_SCOPES = config.copilotApi.copilotScopes;
-  process.env.X_SCENARIO_HEADER = config.copilotApi.scenarioHeader;
+  process.env.WORK_IQ_A2A_ENDPOINT = config.workIq.a2aEndpoint;
+  process.env.WORK_IQ_A2A_CLIENT_ID = config.workIq.a2aClientId;
+  process.env.WORK_IQ_A2A_SCOPES = config.workIq.a2aScopes;
 }
 
 /**
@@ -176,21 +170,80 @@ async function main() {
     .option('--prompts-file <file>', 'JSON file with prompts and expected responses')
     .option('-o, --output <file>', 'output file (JSON, CSV, or HTML)')
     .option('-i, --interactive', 'interactive mode (enter prompts interactively)')
+    .option('--concurrency <number>', 'max prompts to process in parallel (1-5)')
     .option('--m365-agent-id <id>', 'agent ID (overrides env vars and auto-construction)')
     .option('--env <environment>', 'environment name (loads env/.env.<environment>)', 'local')
     .option('--init-only', 'only initialize Python environment, don\'t run evaluations')
     .option('--cache-info', 'show cache information and statistics')
     .option('--cache-clear', 'clear the cache (removes Python runtime and venv)')
     .option('--cache-dir', 'print the cache directory path')
-    .option('--signout', 'sign out and clear cached authentication tokens');
+    .option('--signout', 'sign out and clear cached authentication tokens')
+    .action(() => {
+      // Default command — handled by the main flow below parseAsync()
+    });
+
+  program
+    .command('accept-eula')
+    .description('Accept the End User License Agreement (EULA)')
+    .action(async () => {
+      const config = (await import('../config/default.js')).default;
+      try {
+        await recordAcceptance(config.eula.version);
+        console.log('EULA has been accepted');
+        process.exit(0);
+      } catch (err) {
+        console.error(
+          `⚠️  Unable to persist EULA acceptance: ${err.message}`,
+        );
+        console.error(
+          'Please ensure the directory ~/.m365-copilot-agent-evals/ is writable.',
+        );
+        process.exit(1);
+      }
+    });
 
-  program.parse(process.argv);
+  await program.parseAsync(process.argv);
   const options = program.opts();
   const effectiveLogLevel = resolveLogLevel(options);
   const outputMode = deriveWrapperOutputMode(effectiveLogLevel);
   const wrapperVerbose = outputMode.verbose;
   const wrapperQuiet = outputMode.quiet;
 
+  // === EULA Enforcement Gate ===
+  // Block all commands until EULA is accepted (FR-010, FR-011).
+  // accept-eula subcommand, --help, and --version are already handled
+  // by Commander during program.parse() and exit before reaching here.
+  const config = (await import('../config/default.js')).default;
+  const { accepted, stale } = await checkAcceptance(config.eula.version);
+  if (!accepted) {
+    if (stale) {
+      console.error(
+`==============================================================
+The End User License Agreement (EULA) has been updated.
+Please review the updated terms at:
+${EULA_URL}
+
+To accept the updated EULA, please execute the following command:
+
+    runevals accept-eula
+
+==============================================================`);
+    } else {
+      console.error(
+`==============================================================
+In order to use this tool you must accept the End User License
+Agreement (EULA) found at:
+${EULA_URL}
+
+To accept the EULA, please execute the following command:
+
+    runevals accept-eula
+
+==============================================================`);
+    }
+    process.exit(2);
+  }
+
   // Handle cache commands first (they don't need environment validation or config)
   if (options.cacheInfo) {
     console.log('🗂️  Cache Information\n');
@@ -251,8 +304,7 @@ async function main() {
 
   // === From here on, we're running actual evals - load config and env files ===
 
-  displayUsageTerms();
-  // Load build-time config
+  // Load build-time config (already loaded above for EULA check)
   await setDefaultEnvironmentConstants();
 
   // Load environment files
@@ -322,11 +374,17 @@ async function main() {
     }
   }
 
-  // Resolve agent ID from environment if not explicitly provided via CLI flag
-  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID
+  // Resolve agent ID from environment if not explicitly provided via CLI flag.
+  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID.
+  // Then normalize via shared helper and sync to process.env so downstream
+  // readers (and the python CLI) see the canonical form.
   if (!resolvedAgentId) {
     resolvedAgentId = envVars['M365_AGENT_ID'] || process.env.M365_AGENT_ID;
-    if (resolvedAgentId && !wrapperQuiet) {
+  }
+  resolvedAgentId = normalizeAgentId(resolvedAgentId);
+  if (resolvedAgentId) {
+    process.env.M365_AGENT_ID = resolvedAgentId;
+    if (!options.m365AgentId && !wrapperQuiet) {
       console.log(`🤖 Agent ID: ${resolvedAgentId}`);
     }
   }
@@ -458,6 +516,10 @@ async function main() {
   if (options.prompts && options.prompts.length > 0) {
     pythonArgs.push('--prompts', ...options.prompts);
   }
+
+  if (options.concurrency !== undefined) {
+    pythonArgs.push('--concurrency', String(options.concurrency));
+  }
   
   if (options.expected && options.expected.length > 0) {
     pythonArgs.push('--expected', ...options.expected);

package/src/clients/node-js/config/default.js

@@ -2,24 +2,25 @@
  * Build-time injected default values
  * DO NOT EDIT - This file is auto-generated during build.
  *
- * Generated: 2026-04-01T19:33:48.937Z
+ * Generated: 2026-04-30T18:03:21.788Z
  *
  * @copyright Microsoft Corporation. All rights reserved.
  * @license MIT
  */
 
 export default {
-  copilotApi: {
-    /** Microsoft M365 Evaluation Client ID  */
-    m365EvalClientId: "c678803a-d8e9-4d67-849c-3a8b2d7ba5d3",
+  workIq: {
+    /** Work IQ A2A Endpoint */
+    a2aEndpoint: "https://graph.microsoft.com/rp/workiq",
 
-    /** Copilot OAuth Scopes */
-    copilotScopes: "https://substrate.office.com/sydney/.default",
+    /** Work IQ A2A Client ID */
+    a2aClientId: "ba081686-5d24-4bc6-a0d6-d034ecffed87",
 
-    /** Copilot API Endpoint */
-    copilotApiEndpoint: "https://substrate.office.com/m365Copilot",
-
-    /** Scenario Header for Copilot API */
-    scenarioHeader: "agenticevaluation"
+    /** Work IQ A2A OAuth Scopes */
+    a2aScopes: "Sites.Read.All Mail.Read People.Read.All OnlineMeetingTranscript.Read.All Chat.Read ChannelMessage.Read.All ExternalItem.Read.All"
+  },
+  eula: {
+    /** EULA version string for acceptance tracking */
+    version: "2026-04-01"
   }
 };

package/src/clients/node-js/lib/agent-id.js

@@ -0,0 +1,12 @@
+/**
+ * Normalize an M365 agent ID by appending '.declarativeAgent' when the value
+ * has no '.' segment. Returns the input unchanged when null/undefined/empty
+ * or when it already contains a dot.
+ *
+ * @param {string|null|undefined} id - The raw agent ID value.
+ * @returns {string|null|undefined} The normalized agent ID.
+ */
+export function normalizeAgentId(id) {
+  if (!id) return id;
+  return id.includes('.') ? id : `${id}.declarativeAgent`;
+}

package/src/clients/node-js/lib/env-loader.js

@@ -3,15 +3,15 @@
  * Handles .env.local, .env.local.user, and other env file formats.
  */
 
+import { parse as dotenvParse } from 'dotenv';
 import fs from 'fs';
 import path from 'path';
 
 // Keys that cannot be overridden from .env files (baked in via default.js config)
 const PROTECTED_KEYS = [
-  'M365_EVAL_CLIENT_ID',
-  'COPILOT_API_ENDPOINT',
-  'COPILOT_SCOPES',
-  'X_SCENARIO_HEADER',
+  'WORK_IQ_A2A_ENDPOINT',
+  'WORK_IQ_A2A_CLIENT_ID',
+  'WORK_IQ_A2A_SCOPES',
 ];
 
 // Aliases resolved into M365_AGENT_ID (first match wins)
@@ -21,7 +21,8 @@ const AGENT_ID_ALIASES = [
 
 /**
  * Load environment variables from a .env-style file.
- * Skips blank lines and comments. Protected keys are ignored with a warning.
+ * Uses dotenv.parse() for standards-compliant parsing (handles quoted values,
+ * inline comments, escape sequences). Protected keys are ignored with a warning.
  * Malformed lines (no '=' separator) are skipped with a warning.
  * @param {string} envFilePath - Absolute path to the env file
  * @returns {Object|null} Parsed key-value pairs, or null if file cannot be read
@@ -34,32 +35,25 @@ export function _loadEnvFile(envFilePath) {
   const envVars = {};
   try {
     const content = fs.readFileSync(envFilePath, 'utf-8');
-    const lines = content.split('\n');
 
-    for (const line of lines) {
+    // Pre-scan for malformed lines (no '=') and emit warnings
+    for (const line of content.split('\n')) {
       const trimmedLine = line.trim();
       if (!trimmedLine || trimmedLine.startsWith('#')) {
         continue;
       }
-
-      const eqIndex = trimmedLine.indexOf('=');
-      if (eqIndex === -1) {
+      if (trimmedLine.indexOf('=') === -1) {
         console.warn(
           `⚠️ Ignoring malformed line in env file (missing '='): ${trimmedLine}`
         );
-        continue;
       }
+    }
 
-      const keyName = trimmedLine.slice(0, eqIndex).trim();
-      const value = trimmedLine
-        .slice(eqIndex + 1)
-        .trim()
-        .replace(/^(['"])(.*)\1$/, '$2');
-
-      if (!keyName) {
-        continue;
-      }
+    // Use dotenv.parse() for standards-compliant .env parsing
+    // (handles quoted values, inline comments, escape sequences, export prefix)
+    const parsed = dotenvParse(content);
 
+    for (const [keyName, value] of Object.entries(parsed)) {
       if (PROTECTED_KEYS.includes(keyName)) {
         console.warn(
           `⚠️ Ignoring ${keyName} from .env file (using built-in value)`

package/src/clients/node-js/lib/eula-manager.js

@@ -0,0 +1,78 @@
+/**
+ * EULA acceptance manager
+ *
+ * Manages reading and writing the EULA acceptance marker file at
+ * ~/.m365-copilot-agent-evals/eula-acceptance.json.
+ * This location is independent of the cache directory so acceptance
+ * survives --cache-clear operations.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+
+const EULA_DIR_NAME = '.m365-copilot-agent-evals';
+const EULA_FILE_NAME = 'eula-acceptance.json';
+
+/**
+ * Returns the EULA directory path (~/.m365-copilot-agent-evals/).
+ * @returns {string}
+ */
+export function getEulaDir() {
+  return path.join(os.homedir(), EULA_DIR_NAME);
+}
+
+/**
+ * Returns the full path to the acceptance marker file.
+ * @returns {string}
+ */
+export function getEulaFilePath() {
+  return path.join(getEulaDir(), EULA_FILE_NAME);
+}
+
+/**
+ * Write an acceptance marker for the given EULA version.
+ * Creates the directory if it doesn't exist.
+ * @param {string} version - EULA version string
+ * @returns {Promise<void>}
+ */
+export async function recordAcceptance(version) {
+  const dir = getEulaDir();
+  await fs.mkdir(dir, { recursive: true });
+  const marker = { version, acceptedAt: new Date().toISOString() };
+  await fs.writeFile(
+    getEulaFilePath(),
+    JSON.stringify(marker, null, 2),
+    'utf-8'
+  );
+}
+
+/**
+ * Check whether the EULA has been accepted for the required version.
+ * @param {string} requiredVersion - The version to check against
+ * @returns {Promise<{accepted: boolean, stale: boolean, marker: object|null}>}
+ */
+export async function checkAcceptance(requiredVersion) {
+  const marker = await _readMarker();
+  if (!marker) return { accepted: false, stale: false, marker: null };
+  if (marker.version !== requiredVersion)
+    return { accepted: false, stale: true, marker };
+  return { accepted: true, stale: false, marker };
+}
+
+/**
+ * Read and parse the acceptance marker file.
+ * Returns null if the file is missing, unreadable, or malformed.
+ * Exported with _ prefix for unit testing.
+ * @returns {Promise<object|null>}
+ */
+export async function _readMarker() {
+  try {
+    const raw = await fs.readFile(getEulaFilePath(), 'utf-8');
+    const parsed = JSON.parse(raw);
+    if (!parsed.version || !parsed.acceptedAt) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}

package/src/clients/node-js/lib/progress.js

@@ -605,18 +605,20 @@ export class ProgressReporter {
 
     this.phaseStatuses.set(phaseId, 'failed');
 
-    // Clear current line and display error
-    if (this.isInteractive) {
-      readline.clearLine(process.stdout, 0);
-      readline.cursorTo(process.stdout, 0);
-    }
+    if (!this.options.quiet) {
+      // Clear current line and display error
+      if (this.isInteractive) {
+        readline.clearLine(process.stdout, 0);
+        readline.cursorTo(process.stdout, 0);
+      }
 
-    console.log(`\n❌ Failed: ${phase.name}`);
-    console.log(`\nError: ${error.message}`);
-    console.log(`\nSuggested actions:`);
-    console.log(`  • Check your internet connection`);
-    console.log(`  • If behind a proxy, set HTTP_PROXY/HTTPS_PROXY`);
-    console.log(`  • Run with --verbose for detailed output`);
+      console.error(`\n❌ Failed: ${phase.name}`);
+      console.error(`\nError: ${error.message}`);
+      console.error(`\nSuggested actions:`);
+      console.error(`  • Check your internet connection`);
+      console.error(`  • If behind a proxy, set HTTP_PROXY/HTTPS_PROXY`);
+      console.error(`  • Run with --verbose for detailed output`);
+    }
 
     this.currentPhase = null;
     this.phaseStartTime = null;