@microsoft/m365-copilot-eval 1.1.1-preview.1 → 1.2.0-preview.1

package/src/clients/node-js/bin/runevals.js

@@ -9,6 +9,7 @@ import { ensureVenv, executePythonCli } from '../lib/venv-manager.js';
 import { getCacheStats, clearCache, formatBytes } from '../lib/cache-utils.js';
 import { checkPackageExpiry } from '../lib/expiry-check.js';
 import { ProgressReporter } from '../lib/progress.js';
+import { _loadEnvFile as loadEnvFile, _loadUserEnvOverride } from '../lib/env-loader.js';
 
 // Check package expiry (exits if expired, warns if close to expiry)
 checkPackageExpiry();
@@ -49,58 +50,12 @@ async function setDefaultEnvironmentConstants() {
 }
 
 /**
- * Load environment variables from .env file
+ * Check for required environment variables and provide helpful guidance.
+ * @param {string} envName - Environment name (e.g. 'dev')
+ * @param {boolean} [quiet=false] - Suppress output
+ * @returns {boolean} true if all required vars are present
  */
-function loadEnvFile(envFilePath) {
-  if (!fs.existsSync(envFilePath)) {
-    return null;
-  }
-
-  const envVars = {};
-  try {
-    const content = fs.readFileSync(envFilePath, 'utf-8');
-    const lines = content.split('\n');
-
-    // Protected keys that cannot be overridden from .env files
-    const PROTECTED_KEYS = [
-      'M365_EVAL_CLIENT_ID',
-      'COPILOT_API_ENDPOINT',
-      'COPILOT_SCOPES',
-      'X_SCENARIO_HEADER'
-    ];
-
-    for (const line of lines) {
-      const trimmedLine = line.trim();
-      if (trimmedLine && !trimmedLine.startsWith('#')) {
-        const [key, ...valueParts] = trimmedLine.split('=');
-        const keyName = key.trim();
-        const value = valueParts.join('=').trim().replace(/['"]/g, '');
-
-        // Skip built-in defaults - they cannot be overridden.
-        if (PROTECTED_KEYS.includes(keyName)) {
-          console.warn(`⚠️ Ignoring ${keyName} from .env file (using built-in value)`);
-          continue;
-        }
-
-        if (keyName && value) {
-          envVars[keyName] = value;
-          // Also set in process.env for Python script
-          process.env[keyName] = value;
-        }
-      }
-    }
-  } catch (error) {
-    console.error(`Failed to read environment file: ${error.message}`);
-    return null;
-  }
-
-  return envVars;
-}
-
-/**
- * Check for required environment variables and provide helpful guidance
- */
-function validateEnvironmentVariables(envName, detectedVars, quiet = false) {
+function validateEnvironmentVariables(envName, quiet = false) {
   const required = [
     { key: 'TENANT_ID', description: 'Your Tenant ID' },
     { key: 'AZURE_AI_OPENAI_ENDPOINT', description: 'Azure OpenAI endpoint URL' },
@@ -108,8 +63,8 @@ function validateEnvironmentVariables(envName, detectedVars, quiet = false) {
   ];
 
   const detected = [
-    { key: 'M365_TITLE_ID', description: 'M365 Agent Title ID' }
-  ].filter(opt => process.env[opt.key] || detectedVars[opt.key]);
+    { key: 'M365_AGENT_ID', description: 'M365 Agent ID' }
+  ].filter(opt => process.env[opt.key]);
 
   const missing = required.filter(req => !process.env[req.key]);
 
@@ -121,41 +76,26 @@ function validateEnvironmentVariables(envName, detectedVars, quiet = false) {
   if (!quiet) {
     console.error('\n❌ Missing required environment variables:\n');
 
-    const envFile = envName ? `env/.env.${envName}` : '.env.local or env/env.local';
+    const envFile = envName ? `env/.env.${envName}` : '.env.local or env/.env.local';
     console.error(`Create ${envFile} with:\n`);
 
-  for (const req of missing) {
-    console.error(`  ${req.key}="<your-${req.description.toLowerCase().replace(/\s+/g, '-')}>"`);
-  }
+    for (const req of missing) {
+      console.error(`  ${req.key}="<your-${req.description.toLowerCase().replace(/\s+/g, '-')}>"`);
+    }
 
-  if (detected.length > 0) {
-    console.error(`\n✓ Already detected:`);
-    for (const det of detected) {
-      const value = process.env[det.key] || detectedVars[det.key];
-      console.error(`  ${det.key}="${value}"`);
+    if (detected.length > 0) {
+      console.error(`\n✓ Already detected:`);
+      for (const det of detected) {
+        console.error(`  ${det.key}="${process.env[det.key]}"`);
+      }
     }
-  }
 
-  console.error('\n📖 Setup guide: https://www.npmjs.com/package/@microsoft/m365-copilot-eval?activeTab=readme\n');
+    console.error('\n📖 Setup guide: https://www.npmjs.com/package/@microsoft/m365-copilot-eval?activeTab=readme\n');
   }
 
   return false;
 }
 
-/**
- * Construct agent ID from M365 variables
- */
-function constructAgentId(envVars) {
-  const m365TitleId = envVars['M365_TITLE_ID'];
-
-  if (!m365TitleId) {
-    return null;
-  }
-
-  // Construct agent ID: {M365_TITLE_ID}.declarativeAgent
-  return `${m365TitleId}.declarativeAgent`;
-}
-
 /**
  * Initialize the Python environment (download, venv, pip install)
  * @param {boolean} [verbose=false] - Enable verbose output
@@ -209,7 +149,7 @@ async function main() {
     .option('--prompts-file <file>', 'JSON file with prompts and expected responses')
     .option('-o, --output <file>', 'output file (JSON, CSV, or HTML)')
     .option('-i, --interactive', 'interactive mode (enter prompts interactively)')
-    .option('--agent-id <id>', 'agent ID (overrides env vars and auto-construction)')
+    .option('--m365-agent-id <id>', 'agent ID (overrides env vars and auto-construction)')
     .option('--env <environment>', 'environment name (loads env/.env.<environment>)', 'local')
     .option('--init-only', 'only initialize Python environment, don\'t run evaluations')
     .option('--cache-info', 'show cache information and statistics')
@@ -286,7 +226,7 @@ async function main() {
 
   // Load environment files
   const envVars = {};
-  let resolvedAgentId = options.agentId;
+  let resolvedAgentId = options.m365AgentId;
 
   // Check for .env.local in current directory (ATK projects)
   let localEnvPath = path.join(process.cwd(), '.env.local');
@@ -301,12 +241,12 @@ async function main() {
     localEnvFound = true;
   }
 
-  // If not found, check for env.local in env subfolder
+  // If not found, check for .env.local in env subfolder
   if (!localEnvFound) {
-    localEnvPath = path.join(process.cwd(), 'env', 'env.local');
+    localEnvPath = path.join(process.cwd(), 'env', '.env.local');
     if (fs.existsSync(localEnvPath)) {
       if (!options.quiet && options.verbose) {
-        console.log(`📂 Loading env.local from current directory env folder`);
+        console.log(`📂 Loading .env.local from current directory env folder`);
       }
       const localEnvVars = loadEnvFile(localEnvPath) || {};
       Object.assign(envVars, localEnvVars);
@@ -314,6 +254,9 @@ async function main() {
     }
   }
 
+  // Auto-load .env.local.user as a user-specific override (never checked in, safe for secrets)
+  _loadUserEnvOverride(envVars, options);
+
   if (options.env) {
     // First check current directory's env folder
     let envFilePath = path.join(process.cwd(), 'env', `.env.${options.env}`);
@@ -340,15 +283,7 @@ async function main() {
       }
     }
 
-    if (envFileFound) {
-      // Auto-construct agent ID if not explicitly provided
-      if (!resolvedAgentId) {
-        resolvedAgentId = constructAgentId(envVars);
-        if (resolvedAgentId && !options.quiet) {
-          console.log(`🤖 Agent ID (from M365_TITLE_ID): ${resolvedAgentId}`);
-        }
-      }
-    } else if (options.env !== 'dev') {
+    if (!envFileFound && options.env !== 'dev') {
       // Only warn if non-default env specified
       console.warn(`⚠️  Environment file not found: .env.${options.env}`);
       console.warn(`   Searched in: ${path.join(process.cwd(), 'env')} and ${path.join(__dirname, '..', 'env')}`);
@@ -356,21 +291,17 @@ async function main() {
     }
   }
 
-  // Auto-construct agent ID from loaded env vars if not explicitly provided
-  if (!resolvedAgentId && Object.keys(envVars).length > 0) {
-    resolvedAgentId = constructAgentId(envVars);
+  // Resolve agent ID from environment if not explicitly provided via CLI flag
+  // loadEnvFile already resolved aliases (e.g. M365_TITLE_ID) into M365_AGENT_ID
+  if (!resolvedAgentId) {
+    resolvedAgentId = envVars['M365_AGENT_ID'] || process.env.M365_AGENT_ID;
     if (resolvedAgentId && !options.quiet) {
-      console.log(`🤖 Agent ID (from M365_TITLE_ID): ${resolvedAgentId}`);
+      console.log(`🤖 Agent ID: ${resolvedAgentId}`);
     }
   }
 
-  // Fallback to M365_AGENT_ID env var if still not resolved
-  if (!resolvedAgentId) {
-    resolvedAgentId = process.env.M365_AGENT_ID;
-  }
-
   // Validate required environment variables (always validate, quiet just suppresses output)
-  if (!validateEnvironmentVariables(options.env, envVars, options.quiet)) {
+  if (!validateEnvironmentVariables(options.env, options.quiet)) {
     if (options.quiet) {
       console.error('📖 Setup guide: https://www.npmjs.com/package/@microsoft/m365-copilot-eval?activeTab=readme\n');
     }
@@ -383,7 +314,7 @@ async function main() {
   if (options.verbose) pythonArgs.push('--verbose');
   if (options.quiet) pythonArgs.push('--quiet');
   if (options.interactive) pythonArgs.push('--interactive');
-  if (resolvedAgentId) pythonArgs.push('--agent-id', resolvedAgentId);
+  if (resolvedAgentId) pythonArgs.push('--m365-agent-id', resolvedAgentId);
 
   // Handle signout
   if (options.signout) {

package/src/clients/node-js/config/default.js

@@ -2,7 +2,7 @@
  * Build-time injected default values
  * DO NOT EDIT - This file is auto-generated during build.
  *
- * Generated: 2026-02-04T19:14:08.282Z
+ * Generated: 2026-03-11T17:42:15.925Z
  *
  * @copyright Microsoft Corporation. All rights reserved.
  * @license MIT

package/src/clients/node-js/lib/env-loader.js

@@ -0,0 +1,126 @@
+/**
+ * Environment file loading utilities.
+ * Handles .env.local, .env.local.user, and other env file formats.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+// Keys that cannot be overridden from .env files (baked in via default.js config)
+const PROTECTED_KEYS = [
+  'M365_EVAL_CLIENT_ID',
+  'COPILOT_API_ENDPOINT',
+  'COPILOT_SCOPES',
+  'X_SCENARIO_HEADER',
+];
+
+// Aliases resolved into M365_AGENT_ID (first match wins)
+const AGENT_ID_ALIASES = [
+  { key: 'M365_TITLE_ID', transform: (v) => `${v}.declarativeAgent` },
+];
+
+/**
+ * Load environment variables from a .env-style file.
+ * Skips blank lines and comments. Protected keys are ignored with a warning.
+ * Malformed lines (no '=' separator) are skipped with a warning.
+ * @param {string} envFilePath - Absolute path to the env file
+ * @returns {Object|null} Parsed key-value pairs, or null if file cannot be read
+ */
+export function _loadEnvFile(envFilePath) {
+  if (!fs.existsSync(envFilePath)) {
+    return null;
+  }
+
+  const envVars = {};
+  try {
+    const content = fs.readFileSync(envFilePath, 'utf-8');
+    const lines = content.split('\n');
+
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+      if (!trimmedLine || trimmedLine.startsWith('#')) {
+        continue;
+      }
+
+      const eqIndex = trimmedLine.indexOf('=');
+      if (eqIndex === -1) {
+        console.warn(
+          `⚠️ Ignoring malformed line in env file (missing '='): ${trimmedLine}`
+        );
+        continue;
+      }
+
+      const keyName = trimmedLine.slice(0, eqIndex).trim();
+      const value = trimmedLine
+        .slice(eqIndex + 1)
+        .trim()
+        .replace(/^(['"])(.*)\1$/, '$2');
+
+      if (!keyName) {
+        continue;
+      }
+
+      if (PROTECTED_KEYS.includes(keyName)) {
+        console.warn(
+          `⚠️ Ignoring ${keyName} from .env file (using built-in value)`
+        );
+        continue;
+      }
+
+      if (value) {
+        envVars[keyName] = value;
+        process.env[keyName] = value;
+      }
+    }
+  } catch (error) {
+    console.error(`Failed to read environment file: ${error.message}`);
+    return null;
+  }
+
+  // Resolve agent ID aliases into M365_AGENT_ID (first match wins)
+  if (!envVars['M365_AGENT_ID']) {
+    for (const alias of AGENT_ID_ALIASES) {
+      if (envVars[alias.key]) {
+        const agentId = alias.transform
+          ? alias.transform(envVars[alias.key])
+          : envVars[alias.key];
+        envVars['M365_AGENT_ID'] = agentId;
+        process.env['M365_AGENT_ID'] = agentId;
+        break;
+      }
+    }
+  }
+
+  return envVars;
+}
+
+/**
+ * Auto-load .env.local.user as a user-specific override on top of the base env vars.
+ * Checks cwd first, then env/ subfolder (first found wins).
+ * This mirrors the ATK convention: .env.local is shared/checked-in, .env.local.user holds personal secrets.
+ * @param {Object} envVars - Accumulated env vars object to merge overrides into (mutated in-place)
+ * @param {{ quiet?: boolean, verbose?: boolean }} options - CLI options
+ * @param {string} [cwd] - Working directory to resolve paths from (defaults to process.cwd())
+ * @returns {boolean} Whether a user override file was found and applied
+ */
+export function _loadUserEnvOverride(envVars, options, cwd = process.cwd()) {
+  const userEnvPaths = [
+    path.join(cwd, '.env.local.user'),
+    path.join(cwd, 'env', '.env.local.user'),
+  ];
+
+  for (const userEnvPath of userEnvPaths) {
+    if (fs.existsSync(userEnvPath)) {
+      if (!options.quiet && options.verbose) {
+        console.log(
+          `📂 Loading user overrides from ${path.relative(cwd, userEnvPath)}`
+        );
+      }
+      const userEnvVars = _loadEnvFile(userEnvPath) || {};
+      Object.assign(envVars, userEnvVars);
+      return true;
+    }
+  }
+
+  return false;
+}

package/src/clients/node-js/lib/progress.js

@@ -65,14 +65,18 @@ export function isInteractiveTerminal() {
   if (!process.stdout.isTTY) return false;
 
   // Common CI environment variables
-  if (process.env.CI) return false;
-  if (process.env.GITHUB_ACTIONS) return false;
-  if (process.env.JENKINS_URL) return false;
-  if (process.env.GITLAB_CI) return false;
-  if (process.env.TF_BUILD) return false; // Azure Pipelines
-  if (process.env.CIRCLECI) return false;
-  if (process.env.TRAVIS) return false;
-  if (process.env.BUILDKITE) return false;
+  if (
+    process.env.CI ||
+    process.env.GITHUB_ACTIONS ||
+    process.env.JENKINS_URL ||
+    process.env.GITLAB_CI ||
+    process.env.TF_BUILD || // Azure Pipelines
+    process.env.CIRCLECI ||
+    process.env.TRAVIS ||
+    process.env.BUILDKITE
+  ) {
+    return false;
+  }
 
   return true;
 }
@@ -83,8 +87,7 @@ export function isInteractiveTerminal() {
  * @returns {string} Formatted string (e.g., "45.2 MB")
  */
 export function formatBytes(bytes) {
-  if (bytes === 0) return '0 B';
-  if (bytes < 0) return '0 B';
+  if (bytes <= 0) return '0 B';
 
   const units = ['B', 'KB', 'MB', 'GB', 'TB'];
   const k = 1024;
@@ -302,6 +305,26 @@ export class ProgressReporter {
     }
   }
 
+  /**
+   * Print verbose message with consistent formatting
+   * @param {string} message - Verbose message
+   * @param {Object} [options] - Display options
+   * @param {boolean} [options.clearInteractiveLine] - Clear current interactive line before printing
+   */
+  _printVerboseMessage(message, { clearInteractiveLine = false } = {}) {
+    if (!this.options.verbose || !message) return;
+
+    if (this.isInteractive) {
+      if (clearInteractiveLine) {
+        readline.clearLine(process.stdout, 0);
+        readline.cursorTo(process.stdout, 0);
+      }
+      console.log(`  → ${message}`);
+    } else {
+      console.log(`    → ${message}`);
+    }
+  }
+
   /**
    * Start a new phase
    * @param {string} phaseId - Phase identifier
@@ -470,13 +493,7 @@ export class ProgressReporter {
       }
 
       // Verbose mode: show additional details
-      if (this.options.verbose && message) {
-        if (this.isInteractive) {
-          console.log(`\n  → ${message}`);
-        } else {
-          console.log(`    → ${message}`);
-        }
-      }
+      this._printVerboseMessage(message);
     } else if (
       phase.progressType === 'determinate' &&
       current !== undefined &&
@@ -532,15 +549,7 @@ export class ProgressReporter {
       }
 
       // Verbose mode: show package name
-      if (this.options.verbose && message) {
-        if (this.isInteractive) {
-          readline.clearLine(process.stdout, 0);
-          readline.cursorTo(process.stdout, 0);
-          console.log(`  → ${message}`);
-        } else {
-          console.log(`    → ${message}`);
-        }
-      }
+      this._printVerboseMessage(message, { clearInteractiveLine: true });
     } else if (phase.progressType === 'indeterminate') {
       // For indeterminate phases, spinner handles display in interactive mode
       // In CI mode, output periodically
@@ -553,16 +562,7 @@ export class ProgressReporter {
       }
 
       // Verbose mode: show message if provided
-      if (this.options.verbose && message) {
-        if (this.isInteractive) {
-          // Clear spinner line and print message
-          readline.clearLine(process.stdout, 0);
-          readline.cursorTo(process.stdout, 0);
-          console.log(`  → ${message}`);
-        } else {
-          console.log(`    → ${message}`);
-        }
-      }
+      this._printVerboseMessage(message, { clearInteractiveLine: true });
     }
   }
 

package/src/clients/node-js/lib/python-runtime.js

@@ -230,12 +230,10 @@ export async function ensurePythonRuntime(verbose = false, onProgress) {
   const archivePath = path.join(cacheDir, 'downloads', distribution.filename);
 
   // Determine Python executable path based on platform
-  let pythonExe;
-  if (process.platform === 'win32') {
-    pythonExe = path.join(pythonDir, 'python.exe');
-  } else {
-    pythonExe = path.join(pythonDir, 'bin', 'python3');
-  }
+  const pythonExe =
+    process.platform === 'win32'
+      ? path.join(pythonDir, 'python.exe')
+      : path.join(pythonDir, 'bin', 'python3');
 
   // Check if Python is already installed (silent skip per FR-007)
   try {

package/src/clients/node-js/lib/venv-manager.js

@@ -12,19 +12,12 @@ import { getPythonExecutable, getCacheDir } from './python-runtime.js';
  * @param {boolean} [options.verbose] - Enable verbose output
  * @param {Function} [options.onStdout] - Callback for stdout data
  */
-function execCommand(command, args, options = {}) {
+export function _execCommand(command, args, options = {}) {
   return new Promise((resolve, reject) => {
-    // On Windows, we need shell for proper .exe execution, but passing args
-    // separately with shell:true triggers DEP0190. Join into single command.
-    const isWindows = process.platform === 'win32';
-    const spawnArgs = isWindows ? [] : args;
-    const spawnCommand = isWindows
-      ? `"${command}" ${args.map((a) => `"${a}"`).join(' ')}`
-      : command;
-
-    const proc = spawn(spawnCommand, spawnArgs, {
+    const proc = spawn(command, args, {
       ...options,
-      shell: isWindows,
+      shell: false,
+      windowsVerbatimArguments: false,
     });
 
     let stdout = '';
@@ -109,7 +102,7 @@ async function createVenv(pythonExe, venvDir, verbose = false, onProgress) {
 
   await fs.mkdir(venvDir, { recursive: true });
 
-  await execCommand(pythonExe, ['-m', 'venv', venvDir], { verbose });
+  await _execCommand(pythonExe, ['-m', 'venv', venvDir], { verbose });
 
   if (verbose && !onProgress) {
     console.log('Virtual environment created ✓');
@@ -123,7 +116,7 @@ async function createVenv(pythonExe, venvDir, verbose = false, onProgress) {
  * @param {boolean} [verbose=false] - Enable verbose output
  * @param {Function} [onProgress] - Optional progress callback
  */
-async function installRequirements(
+export async function _installRequirements(
   venvDir,
   requirementsPath,
   verbose = false,
@@ -140,7 +133,7 @@ async function installRequirements(
   }
 
   // Upgrade pip first (use python -m pip to avoid Windows file locking issues)
-  await execCommand(venvPython, ['-m', 'pip', 'install', '--upgrade', 'pip'], {
+  await _execCommand(venvPython, ['-m', 'pip', 'install', '--upgrade', 'pip'], {
     verbose,
   });
 
@@ -150,8 +143,16 @@ async function installRequirements(
   let cachedCount = 0;
   let pipPhase = 'collecting'; // 'collecting', 'resolving', 'downloading', 'installing'
 
-  // Install requirements with hash checking if available
-  const args = ['-m', 'pip', 'install', '-r', requirementsPath];
+  // Install requirements with optimization flags
+  const args = [
+    '-m',
+    'pip',
+    'install',
+    '--prefer-binary', // Prefer binary wheels over building from source
+    '--no-compile', // Defer bytecode compilation to after installation for faster install time
+    '-r',
+    requirementsPath,
+  ];
 
   // Support proxy and certificate configuration
   if (process.env.PIP_CERT) {
@@ -232,7 +233,48 @@ async function installRequirements(
       }
     : undefined;
 
-  await execCommand(venvPython, args, { verbose, onStdout });
+  await _execCommand(venvPython, args, { verbose, onStdout });
+
+  // Compile bytecode after installation to optimize first run performance
+  // This compensates for --no-compile flag and eliminates ~1 min delay on first eval
+  if (verbose) {
+    console.log('Compiling bytecode for faster first run...');
+  }
+
+  try {
+    // Find site-packages directory based on platform
+    let sitePackagesDir;
+    if (process.platform === 'win32') {
+      sitePackagesDir = path.join(venvDir, 'Lib', 'site-packages');
+    } else {
+      // shell: false means globs aren't expanded by the shell — resolve in Node
+      const pattern = path.join(venvDir, 'lib', 'python*', 'site-packages');
+      const matches = await Array.fromAsync(fs.glob(pattern));
+      sitePackagesDir = matches[0];
+    }
+
+    if (!sitePackagesDir) {
+      // Skip pre-compilation; bytecode will be compiled lazily on first import
+      return;
+    }
+
+    await _execCommand(
+      venvPython,
+      ['-m', 'compileall', '-q', '-j', '0', sitePackagesDir],
+      { verbose: false }
+    );
+
+    if (verbose) {
+      console.log('Bytecode compilation complete');
+    }
+  } catch {
+    // Non-fatal: bytecode will be compiled on first import if this fails
+    if (verbose) {
+      console.warn(
+        'Warning: Bytecode compilation failed, will compile on first import'
+      );
+    }
+  }
 
   if (!onProgress) {
     console.log('Dependencies installed ✓');
@@ -320,7 +362,7 @@ export async function ensureVenv(
   // Install dependencies phase
   onProgress?.({ type: 'start', phaseId: 'deps' });
   try {
-    await installRequirements(venvDir, requirementsPath, verbose, onProgress);
+    await _installRequirements(venvDir, requirementsPath, verbose, onProgress);
     onProgress?.({ type: 'complete', phaseId: 'deps' });
   } catch (error) {
     onProgress?.({ type: 'error', phaseId: 'deps', error });