@microsoft/m365-copilot-eval 1.2.1-preview.1 → 1.4.0-preview.1

package/src/clients/node-js/bin/runevals.js

@@ -8,8 +8,10 @@ import { ensurePythonRuntime, getCacheDir } from '../lib/python-runtime.js';
 import { ensureVenv, executePythonCli } from '../lib/venv-manager.js';
 import { getCacheStats, clearCache, formatBytes } from '../lib/cache-utils.js';
 import { checkPackageExpiry } from '../lib/expiry-check.js';
+import { recordAcceptance, checkAcceptance } from '../lib/eula-manager.js';
 import { ProgressReporter } from '../lib/progress.js';
 import { _loadEnvFile as loadEnvFile, _loadUserEnvOverride } from '../lib/env-loader.js';
+import { normalizeAgentId } from '../lib/agent-id.js';
 
 // Check package expiry (exits if expired, warns if close to expiry)
 checkPackageExpiry();
@@ -22,20 +24,13 @@ const packageJsonPath = path.join(__dirname, '..', '..', '..', '..', 'package.js
 const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
 const VERSION = packageJson.version;
 
+const EULA_URL = 'https://aka.ms/evaltoolterms';
+
 // Path to Python CLI and requirements
 const PYTHON_CLI_DIR = path.join(__dirname, '..', '..', 'cli');
 const MAIN_SCRIPT = path.join(PYTHON_CLI_DIR, 'main.py');
 const REQUIREMENTS_FILE = path.join(PYTHON_CLI_DIR, 'requirements.txt');
 
-/**
- * Display usage terms notice
- * Called before running evaluations (but not for --init-only, cache commands, or --signout)
- * This notice MUST be displayed even in quiet mode per legal requirements (FR-006)
- */
-function displayUsageTerms() {
-  console.log('By using this tool, you agree to the Terms of Use: https://aka.ms/evaltoolterms\n');
-}
-
 /**
  * Set default environment constants that cannot be overridden
  * This ensures these values are always set regardless of .env files
@@ -52,10 +47,10 @@ async function setDefaultEnvironmentConstants() {
 /**
  * Check for required environment variables and provide helpful guidance.
  * @param {string} envName - Environment name (e.g. 'dev')
- * @param {boolean} [quiet=false] - Suppress output
+ * @param {boolean} [suppressOutput=false] - Suppress guidance output
  * @returns {boolean} true if all required vars are present
  */
-function validateEnvironmentVariables(envName, quiet = false) {
+function validateEnvironmentVariables(envName, suppressOutput = false) {
   const required = [
     { key: 'TENANT_ID', description: 'Your Tenant ID' },
     { key: 'AZURE_AI_OPENAI_ENDPOINT', description: 'Azure OpenAI endpoint URL' },
@@ -72,8 +67,8 @@ function validateEnvironmentVariables(envName, quiet = false) {
     return true; // All required vars present
   }
 
-  // Show error with helpful guidance (skip output in quiet mode, but still return false)
-  if (!quiet) {
+  // Show error with helpful guidance unless output is suppressed.
+  if (!suppressOutput) {
     console.error('\n❌ Missing required environment variables:\n');
 
     const envFile = envName ? `env/.env.${envName}` : '.env.local or env/.env.local';
@@ -96,6 +91,34 @@ function validateEnvironmentVariables(envName, quiet = false) {
   return false;
 }
 
+/**
+ * Resolve canonical log level for Python CLI forwarding.
+ */
+function resolveLogLevel(options) {
+  const allowedLevels = new Set(['debug', 'info', 'warning', 'error']);
+  const rawLogLevel =
+    options.logLevel === true || options.logLevel === undefined
+      ? undefined
+      : String(options.logLevel).toLowerCase();
+  const effectiveLogLevel = rawLogLevel || 'info';
+
+  if (!allowedLevels.has(effectiveLogLevel)) {
+    console.error(
+      `❌ Invalid --log-level value: ${effectiveLogLevel}. Supported values: debug, info, warning, error.`
+    );
+    process.exit(2);
+  }
+
+  return effectiveLogLevel;
+}
+
+function deriveWrapperOutputMode(effectiveLogLevel) {
+  return {
+    verbose: effectiveLogLevel === 'debug',
+    quiet: effectiveLogLevel === 'warning' || effectiveLogLevel === 'error',
+  };
+}
+
 /**
  * Initialize the Python environment (download, venv, pip install)
  * @param {boolean} [verbose=false] - Enable verbose output
@@ -126,7 +149,7 @@ async function initializePythonEnvironment(verbose = false, quiet = false) {
     console.error('  - Check your internet connection');
     console.error('  - If behind a proxy, set HTTP_PROXY/HTTPS_PROXY environment variables');
     console.error('  - For SSL issues, set NODE_EXTRA_CA_CERTS or PIP_CERT');
-    console.error('  - Run with --verbose for detailed output');
+    console.error('  - Run with --log-level debug for detailed output');
 
     process.exit(1);
   }
@@ -142,23 +165,85 @@ async function main() {
     .name('runevals')
     .description('M365 Copilot Agent Evaluations CLI - Zero-config Python evaluation tool')
     .version(VERSION)
-    .option('-v, --verbose', 'verbose output (shows detailed processing steps)')
-    .option('-q, --quiet', 'quiet mode (minimal output)')
+    .option('--log-level [level]', 'log level for diagnostics: debug, info, warning, error (bare flag resolves to info)')
     .option('--prompts <prompts...>', 'prompts to evaluate')
     .option('--expected <responses...>', 'expected responses')
     .option('--prompts-file <file>', 'JSON file with prompts and expected responses')
     .option('-o, --output <file>', 'output file (JSON, CSV, or HTML)')
     .option('-i, --interactive', 'interactive mode (enter prompts interactively)')
+    .option('--concurrency <number>', 'max prompts to process in parallel (1-5)')
     .option('--m365-agent-id <id>', 'agent ID (overrides env vars and auto-construction)')
     .option('--env <environment>', 'environment name (loads env/.env.<environment>)', 'local')
     .option('--init-only', 'only initialize Python environment, don\'t run evaluations')
     .option('--cache-info', 'show cache information and statistics')
     .option('--cache-clear', 'clear the cache (removes Python runtime and venv)')
     .option('--cache-dir', 'print the cache directory path')
-    .option('--signout', 'sign out and clear cached authentication tokens');
+    .option('--signout', 'sign out and clear cached authentication tokens')
+    .action(() => {
+      // Default command — handled by the main flow below parseAsync()
+    });
+
+  program
+    .command('accept-eula')
+    .description('Accept the End User License Agreement (EULA)')
+    .action(async () => {
+      const config = (await import('../config/default.js')).default;
+      try {
+        await recordAcceptance(config.eula.version);
+        console.log('EULA has been accepted');
+        process.exit(0);
+      } catch (err) {
+        console.error(
+          `⚠️  Unable to persist EULA acceptance: ${err.message}`,
+        );
+        console.error(
+          'Please ensure the directory ~/.m365-copilot-agent-evals/ is writable.',
+        );
+        process.exit(1);
+      }
+    });
 
-  program.parse(process.argv);
+  await program.parseAsync(process.argv);
   const options = program.opts();
+  const effectiveLogLevel = resolveLogLevel(options);
+  const outputMode = deriveWrapperOutputMode(effectiveLogLevel);
+  const wrapperVerbose = outputMode.verbose;
+  const wrapperQuiet = outputMode.quiet;
+
+  // === EULA Enforcement Gate ===
+  // Block all commands until EULA is accepted (FR-010, FR-011).
+  // accept-eula subcommand, --help, and --version are already handled
+  // by Commander during program.parse() and exit before reaching here.
+  const config = (await import('../config/default.js')).default;
+  const { accepted, stale } = await checkAcceptance(config.eula.version);
+  if (!accepted) {
+    if (stale) {
+      console.error(
+`==============================================================
+The End User License Agreement (EULA) has been updated.
+Please review the updated terms at:
+${EULA_URL}
+
+To accept the updated EULA, please execute the following command:
+
+    runevals accept-eula
+
+==============================================================`);
+    } else {
+      console.error(
+`==============================================================
+In order to use this tool you must accept the End User License
+Agreement (EULA) found at:
+${EULA_URL}
+
+To accept the EULA, please execute the following command:
+
+    runevals accept-eula
+
+==============================================================`);
+    }
+    process.exit(2);
+  }
 
   // Handle cache commands first (they don't need environment validation or config)
   if (options.cacheInfo) {
@@ -183,7 +268,7 @@ async function main() {
   
   if (options.cacheClear) {
     console.log('🗑️  Clearing cache...\n');
-    const success = await clearCache(options.verbose);
+    const success = await clearCache(wrapperVerbose);
     
     if (success) {
       console.log('✅ Cache cleared successfully!');
@@ -203,14 +288,14 @@ async function main() {
   
   // Initialize Python environment (do this early for --init-only)
   // Skip env file loading for --init-only since it's not needed
-  if (!options.quiet) {
+  if (!wrapperQuiet) {
     console.log('🚀 M365 Copilot Agent Evaluations CLI\n');
   }
-  await initializePythonEnvironment(options.verbose, options.quiet);
+  await initializePythonEnvironment(wrapperVerbose, wrapperQuiet);
 
   // If --init-only, stop here (no config or env files needed)
   if (options.initOnly) {
-    if (!options.quiet) {
+    if (!wrapperQuiet) {
       console.log('\n✅ Python environment initialized successfully!\n');
       console.log('⚠️  Note: Configure environment variables before running evaluations.');
       console.log('📖 See README.md for complete setup guide.\n');
@@ -220,8 +305,7 @@ async function main() {
 
   // === From here on, we're running actual evals - load config and env files ===
 
-  displayUsageTerms();
-  // Load build-time config
+  // Load build-time config (already loaded above for EULA check)
   await setDefaultEnvironmentConstants();
 
   // Load environment files
@@ -233,7 +317,7 @@ async function main() {
   let localEnvFound = false;
 
   if (fs.existsSync(localEnvPath)) {
-    if (!options.quiet && options.verbose) {
+    if (!wrapperQuiet && wrapperVerbose) {
       console.log(`📂 Loading .env.local from current directory`);
     }
     const localEnvVars = loadEnvFile(localEnvPath) || {};
@@ -245,7 +329,7 @@ async function main() {
   if (!localEnvFound) {
     localEnvPath = path.join(process.cwd(), 'env', '.env.local');
     if (fs.existsSync(localEnvPath)) {
-      if (!options.quiet && options.verbose) {
+      if (!wrapperQuiet && wrapperVerbose) {
         console.log(`📂 Loading .env.local from current directory env folder`);
       }
       const localEnvVars = loadEnvFile(localEnvPath) || {};
@@ -263,7 +347,7 @@ async function main() {
     let envFileFound = false;
 
     if (fs.existsSync(envFilePath)) {
-      if (!options.quiet) {
+      if (!wrapperQuiet) {
         console.log(`📂 Loading environment: ${options.env} from current directory env folder`);
       }
       const fileEnvVars = loadEnvFile(envFilePath) || {};
@@ -274,7 +358,7 @@ async function main() {
       envFilePath = path.join(__dirname, '..', 'env', `.env.${options.env}`);
 
       if (fs.existsSync(envFilePath)) {
-        if (!options.quiet) {
+        if (!wrapperQuiet) {
           console.log(`📂 Loading environment: ${options.env} from package env folder`);
         }
         const fileEnvVars = loadEnvFile(envFilePath) || {};
@@ -291,18 +375,24 @@ async function main() {
     }
   }
 
-  // Resolve agent ID from environment if not explicitly provided via CLI flag
-  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID
+  // Resolve agent ID from environment if not explicitly provided via CLI flag.
+  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID.
+  // Then normalize via shared helper and sync to process.env so downstream
+  // readers (and the python CLI) see the canonical form.
   if (!resolvedAgentId) {
     resolvedAgentId = envVars['M365_AGENT_ID'] || process.env.M365_AGENT_ID;
-    if (resolvedAgentId && !options.quiet) {
+  }
+  resolvedAgentId = normalizeAgentId(resolvedAgentId);
+  if (resolvedAgentId) {
+    process.env.M365_AGENT_ID = resolvedAgentId;
+    if (!options.m365AgentId && !wrapperQuiet) {
       console.log(`🤖 Agent ID: ${resolvedAgentId}`);
     }
   }
 
   // Validate required environment variables (always validate, quiet just suppresses output)
-  if (!validateEnvironmentVariables(options.env, options.quiet)) {
-    if (options.quiet) {
+  if (!validateEnvironmentVariables(options.env, wrapperQuiet)) {
+    if (wrapperQuiet) {
       console.error('📖 Setup guide: https://www.npmjs.com/package/@microsoft/m365-copilot-eval?activeTab=readme\n');
     }
     process.exit(1);
@@ -311,8 +401,7 @@ async function main() {
   // Build arguments to pass to Python CLI
   const pythonArgs = [];
 
-  if (options.verbose) pythonArgs.push('--verbose');
-  if (options.quiet) pythonArgs.push('--quiet');
+  pythonArgs.push('--log-level', effectiveLogLevel);
   if (options.interactive) pythonArgs.push('--interactive');
   if (resolvedAgentId) pythonArgs.push('--m365-agent-id', resolvedAgentId);
 
@@ -327,7 +416,7 @@ async function main() {
       console.log('✅ Successfully signed out and cleared cached authentication tokens!');
     } catch (error) {
       console.error('❌ Failed to sign out:', error.message);
-      if (options.verbose) {
+      if (wrapperVerbose) {
         console.error('\nFull error:', error);
       }
       process.exit(1);
@@ -350,7 +439,7 @@ async function main() {
         const candidatePath = path.join(location.dir, filename);
         if (fs.existsSync(candidatePath)) {
           promptsFile = candidatePath;
-          if (!options.quiet) {
+          if (!wrapperQuiet) {
             const displayPath = candidatePath.startsWith(process.cwd()) 
               ? './' + path.relative(process.cwd(), candidatePath)
               : candidatePath;
@@ -363,7 +452,7 @@ async function main() {
     }
     
     // If no prompts file found, offer to create starter file
-    if (!promptsFile && !options.quiet) {
+    if (!promptsFile && !wrapperQuiet) {
       console.log('\n⚠️  No prompts file found in current directory or ./evals/\n');
       
       // Use readline for interactive prompt
@@ -413,7 +502,7 @@ async function main() {
     const evalsDir = path.join(process.cwd(), '.evals');
     if (!fs.existsSync(evalsDir)) {
       fs.mkdirSync(evalsDir, { recursive: true });
-      if (options.verbose) {
+      if (wrapperVerbose) {
         console.log(`Created .evals directory`);
       }
     }
@@ -428,13 +517,17 @@ async function main() {
   if (options.prompts && options.prompts.length > 0) {
     pythonArgs.push('--prompts', ...options.prompts);
   }
+
+  if (options.concurrency !== undefined) {
+    pythonArgs.push('--concurrency', String(options.concurrency));
+  }
   
   if (options.expected && options.expected.length > 0) {
     pythonArgs.push('--expected', ...options.expected);
   }
   
   // Execute Python CLI with working directory set to Python CLI directory
-  if (!options.quiet) {
+  if (!wrapperQuiet) {
     console.log('\n📊 Running evaluations...\n');
     console.log('─────────────────────────────────────────────────────────────\n');
   }
@@ -442,7 +535,7 @@ async function main() {
   try {
     await executePythonCli(MAIN_SCRIPT, pythonArgs, { cwd: PYTHON_CLI_DIR });
     
-    if (!options.quiet) {
+    if (!wrapperQuiet) {
       console.log('\n─────────────────────────────────────────────────────────────\n');
       console.log('✓ Evals completed successfully!');
       if (outputFile) {

package/src/clients/node-js/config/default.js

@@ -2,7 +2,7 @@
  * Build-time injected default values
  * DO NOT EDIT - This file is auto-generated during build.
  *
- * Generated: 2026-03-23T18:11:37.402Z
+ * Generated: 2026-04-22T20:44:41.713Z
  *
  * @copyright Microsoft Corporation. All rights reserved.
  * @license MIT
@@ -21,5 +21,9 @@ export default {
 
     /** Scenario Header for Copilot API */
     scenarioHeader: "agenticevaluation"
+  },
+  eula: {
+    /** EULA version string for acceptance tracking */
+    version: "2026-04-01"
   }
 };

package/src/clients/node-js/lib/agent-id.js

@@ -0,0 +1,12 @@
+/**
+ * Normalize an M365 agent ID by appending '.declarativeAgent' when the value
+ * has no '.' segment. Returns the input unchanged when null/undefined/empty
+ * or when it already contains a dot.
+ *
+ * @param {string|null|undefined} id - The raw agent ID value.
+ * @returns {string|null|undefined} The normalized agent ID.
+ */
+export function normalizeAgentId(id) {
+  if (!id) return id;
+  return id.includes('.') ? id : `${id}.declarativeAgent`;
+}

package/src/clients/node-js/lib/env-loader.js

@@ -3,6 +3,7 @@
  * Handles .env.local, .env.local.user, and other env file formats.
  */
 
+import { parse as dotenvParse } from 'dotenv';
 import fs from 'fs';
 import path from 'path';
 
@@ -21,7 +22,8 @@ const AGENT_ID_ALIASES = [
 
 /**
  * Load environment variables from a .env-style file.
- * Skips blank lines and comments. Protected keys are ignored with a warning.
+ * Uses dotenv.parse() for standards-compliant parsing (handles quoted values,
+ * inline comments, escape sequences). Protected keys are ignored with a warning.
  * Malformed lines (no '=' separator) are skipped with a warning.
  * @param {string} envFilePath - Absolute path to the env file
  * @returns {Object|null} Parsed key-value pairs, or null if file cannot be read
@@ -34,32 +36,25 @@ export function _loadEnvFile(envFilePath) {
   const envVars = {};
   try {
     const content = fs.readFileSync(envFilePath, 'utf-8');
-    const lines = content.split('\n');
 
-    for (const line of lines) {
+    // Pre-scan for malformed lines (no '=') and emit warnings
+    for (const line of content.split('\n')) {
       const trimmedLine = line.trim();
       if (!trimmedLine || trimmedLine.startsWith('#')) {
         continue;
       }
-
-      const eqIndex = trimmedLine.indexOf('=');
-      if (eqIndex === -1) {
+      if (trimmedLine.indexOf('=') === -1) {
         console.warn(
           `⚠️ Ignoring malformed line in env file (missing '='): ${trimmedLine}`
         );
-        continue;
       }
+    }
 
-      const keyName = trimmedLine.slice(0, eqIndex).trim();
-      const value = trimmedLine
-        .slice(eqIndex + 1)
-        .trim()
-        .replace(/^(['"])(.*)\1$/, '$2');
-
-      if (!keyName) {
-        continue;
-      }
+    // Use dotenv.parse() for standards-compliant .env parsing
+    // (handles quoted values, inline comments, escape sequences, export prefix)
+    const parsed = dotenvParse(content);
 
+    for (const [keyName, value] of Object.entries(parsed)) {
       if (PROTECTED_KEYS.includes(keyName)) {
         console.warn(
           `⚠️ Ignoring ${keyName} from .env file (using built-in value)`

package/src/clients/node-js/lib/eula-manager.js

@@ -0,0 +1,78 @@
+/**
+ * EULA acceptance manager
+ *
+ * Manages reading and writing the EULA acceptance marker file at
+ * ~/.m365-copilot-agent-evals/eula-acceptance.json.
+ * This location is independent of the cache directory so acceptance
+ * survives --cache-clear operations.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+
+const EULA_DIR_NAME = '.m365-copilot-agent-evals';
+const EULA_FILE_NAME = 'eula-acceptance.json';
+
+/**
+ * Returns the EULA directory path (~/.m365-copilot-agent-evals/).
+ * @returns {string}
+ */
+export function getEulaDir() {
+  return path.join(os.homedir(), EULA_DIR_NAME);
+}
+
+/**
+ * Returns the full path to the acceptance marker file.
+ * @returns {string}
+ */
+export function getEulaFilePath() {
+  return path.join(getEulaDir(), EULA_FILE_NAME);
+}
+
+/**
+ * Write an acceptance marker for the given EULA version.
+ * Creates the directory if it doesn't exist.
+ * @param {string} version - EULA version string
+ * @returns {Promise<void>}
+ */
+export async function recordAcceptance(version) {
+  const dir = getEulaDir();
+  await fs.mkdir(dir, { recursive: true });
+  const marker = { version, acceptedAt: new Date().toISOString() };
+  await fs.writeFile(
+    getEulaFilePath(),
+    JSON.stringify(marker, null, 2),
+    'utf-8'
+  );
+}
+
+/**
+ * Check whether the EULA has been accepted for the required version.
+ * @param {string} requiredVersion - The version to check against
+ * @returns {Promise<{accepted: boolean, stale: boolean, marker: object|null}>}
+ */
+export async function checkAcceptance(requiredVersion) {
+  const marker = await _readMarker();
+  if (!marker) return { accepted: false, stale: false, marker: null };
+  if (marker.version !== requiredVersion)
+    return { accepted: false, stale: true, marker };
+  return { accepted: true, stale: false, marker };
+}
+
+/**
+ * Read and parse the acceptance marker file.
+ * Returns null if the file is missing, unreadable, or malformed.
+ * Exported with _ prefix for unit testing.
+ * @returns {Promise<object|null>}
+ */
+export async function _readMarker() {
+  try {
+    const raw = await fs.readFile(getEulaFilePath(), 'utf-8');
+    const parsed = JSON.parse(raw);
+    if (!parsed.version || !parsed.acceptedAt) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}

package/src/clients/node-js/lib/progress.js

@@ -605,18 +605,20 @@ export class ProgressReporter {
 
     this.phaseStatuses.set(phaseId, 'failed');
 
-    // Clear current line and display error
-    if (this.isInteractive) {
-      readline.clearLine(process.stdout, 0);
-      readline.cursorTo(process.stdout, 0);
-    }
+    if (!this.options.quiet) {
+      // Clear current line and display error
+      if (this.isInteractive) {
+        readline.clearLine(process.stdout, 0);
+        readline.cursorTo(process.stdout, 0);
+      }
 
-    console.log(`\n❌ Failed: ${phase.name}`);
-    console.log(`\nError: ${error.message}`);
-    console.log(`\nSuggested actions:`);
-    console.log(`  • Check your internet connection`);
-    console.log(`  • If behind a proxy, set HTTP_PROXY/HTTPS_PROXY`);
-    console.log(`  • Run with --verbose for detailed output`);
+      console.error(`\n❌ Failed: ${phase.name}`);
+      console.error(`\nError: ${error.message}`);
+      console.error(`\nSuggested actions:`);
+      console.error(`  • Check your internet connection`);
+      console.error(`  • If behind a proxy, set HTTP_PROXY/HTTPS_PROXY`);
+      console.error(`  • Run with --verbose for detailed output`);
+    }
 
     this.currentPhase = null;
     this.phaseStartTime = null;