@microsoft/m365-copilot-eval 1.0.1-preview.1

package/src/clients/node-js/bin/runevals.js

@@ -0,0 +1,505 @@
+#!/usr/bin/env node
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+import fs from 'fs';
+import { Command } from 'commander';
+import { ensurePythonRuntime, getCacheDir } from '../lib/python-runtime.js';
+import { ensureVenv, executePythonCli } from '../lib/venv-manager.js';
+import { getCacheStats, clearCache, formatBytes } from '../lib/cache-utils.js';
+import { checkPackageExpiry } from '../lib/expiry-check.js';
+import config from '../config/default.js';
+
+// Check package expiry (exits if expired, warns if close to expiry)
+checkPackageExpiry();
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Read version from package.json
+const packageJsonPath = path.join(__dirname, '..', '..', '..', '..', 'package.json');
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+const VERSION = packageJson.version;
+
+// Path to Python CLI and requirements
+const PYTHON_CLI_DIR = path.join(__dirname, '..', '..', 'cli');
+const MAIN_SCRIPT = path.join(PYTHON_CLI_DIR, 'main.py');
+const REQUIREMENTS_FILE = path.join(PYTHON_CLI_DIR, 'requirements.txt');
+
+/**
+ * Set default environment constants that cannot be overridden
+ * This ensures these values are always set regardless of .env files
+ */
+function setDefaultEnvironmentConstants() {
+  process.env.M365_EVAL_CLIENT_ID = config.copilotApi.m365EvalClientId;
+  process.env.COPILOT_API_ENDPOINT = config.copilotApi.copilotApiEndpoint;
+  process.env.COPILOT_SCOPES = config.copilotApi.copilotScopes;
+  process.env.X_SCENARIO_HEADER = config.copilotApi.scenarioHeader;
+}
+
+/**
+ * Load environment variables from .env file
+ */
+function loadEnvFile(envFilePath) {
+  if (!fs.existsSync(envFilePath)) {
+    return null;
+  }
+
+  const envVars = {};
+  try {
+    const content = fs.readFileSync(envFilePath, 'utf-8');
+    const lines = content.split('\n');
+
+    // Protected keys that cannot be overridden from .env files
+    const PROTECTED_KEYS = [
+      'M365_EVAL_CLIENT_ID',
+      'COPILOT_API_ENDPOINT',
+      'COPILOT_SCOPES',
+      'X_SCENARIO_HEADER'
+    ];
+
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+      if (trimmedLine && !trimmedLine.startsWith('#')) {
+        const [key, ...valueParts] = trimmedLine.split('=');
+        const keyName = key.trim();
+        const value = valueParts.join('=').trim().replace(/['"]/g, '');
+
+        // Skip built-in defaults - they cannot be overridden.
+        if (PROTECTED_KEYS.includes(keyName)) {
+          console.warn(`⚠️ Ignoring ${keyName} from .env file (using built-in value)`);
+          continue;
+        }
+
+        if (keyName && value) {
+          envVars[keyName] = value;
+          // Also set in process.env for Python script
+          process.env[keyName] = value;
+        }
+      }
+    }
+  } catch (error) {
+    console.error(`Failed to read environment file: ${error.message}`);
+    return null;
+  }
+
+  return envVars;
+}
+
+/**
+ * Check for required environment variables and provide helpful guidance
+ */
+function validateEnvironmentVariables(envName, detectedVars) {
+  const required = [
+    { key: 'TENANT_ID', description: 'Your Tenant ID' },
+    { key: 'AZURE_AI_OPENAI_ENDPOINT', description: 'Azure OpenAI endpoint URL' },
+    { key: 'AZURE_AI_API_KEY', description: 'Azure OpenAI API key' }
+  ];
+
+  const detected = [
+    { key: 'M365_TITLE_ID', description: 'M365 Agent Title ID' }
+  ].filter(opt => process.env[opt.key] || detectedVars[opt.key]);
+
+  const missing = required.filter(req => !process.env[req.key]);
+
+  if (missing.length === 0) {
+    return true; // All required vars present
+  }
+
+  // Show error with helpful guidance
+  console.error('\n❌ Missing required environment variables:\n');
+
+  const envFile = envName ? `env/.env.${envName}` : '.env.local or env/env.local';
+  console.error(`Create ${envFile} with:\n`);
+
+  for (const req of missing) {
+    console.error(`  ${req.key}="<your-${req.description.toLowerCase().replace(/\s+/g, '-')}>"`);
+  }
+
+  if (detected.length > 0) {
+    console.error(`\n✓ Already detected:`);
+    for (const det of detected) {
+      const value = process.env[det.key] || detectedVars[det.key];
+      console.error(`  ${det.key}="${value}"`);
+    }
+  }
+
+  console.error(`\nThen run: npx runevals${envName ? ` --env ${envName}` : ''}\n`);
+  console.error('📖 See README.md for complete setup guide.\n');
+
+  return false;
+}
+
+/**
+ * Construct agent ID from M365 variables
+ */
+function constructAgentId(envVars) {
+  const m365TitleId = envVars['M365_TITLE_ID'];
+
+  if (!m365TitleId) {
+    return null;
+  }
+
+  // Construct agent ID: {M365_TITLE_ID}.declarativeAgent
+  return `${m365TitleId}.declarativeAgent`;
+}
+
+/**
+ * Initialize the Python environment (download, venv, pip install)
+ */
+async function initializePythonEnvironment(verbose = false) {
+  try {
+    // Step 1: Ensure Python runtime is available
+    await ensurePythonRuntime(verbose);
+    
+    // Step 2: Ensure venv with dependencies is set up
+    await ensureVenv(REQUIREMENTS_FILE, verbose);
+    
+  } catch (error) {
+    console.error('\n❌ Failed to initialize Python environment:');
+    console.error(error.message);
+    
+    if (verbose) {
+      console.error('\nFull error:', error);
+    }
+    
+    console.error('\nTroubleshooting:');
+    console.error('  - Check your internet connection');
+    console.error('  - If behind a proxy, set HTTP_PROXY/HTTPS_PROXY environment variables');
+    console.error('  - For SSL issues, set NODE_EXTRA_CA_CERTS or PIP_CERT');
+    console.error('  - Run with --verbose for detailed output');
+    
+    process.exit(1);
+  }
+}
+
+/**
+ * Main CLI entry point
+ */
+async function main() {
+  // Set default environment constants.
+  setDefaultEnvironmentConstants();
+  
+  const program = new Command();
+  
+  program
+    .name('runevals')
+    .description('M365 Copilot Agent Evaluations CLI - Zero-config Python evaluation tool')
+    .version(VERSION)
+    .option('-v, --verbose', 'verbose output (shows detailed processing steps)')
+    .option('-q, --quiet', 'quiet mode (minimal output)')
+    .option('--prompts <prompts...>', 'prompts to evaluate')
+    .option('--expected <responses...>', 'expected responses')
+    .option('--prompts-file <file>', 'JSON file with prompts and expected responses')
+    .option('-o, --output <file>', 'output file (JSON, CSV, or HTML)')
+    .option('-i, --interactive', 'interactive mode (enter prompts interactively)')
+    .option('--agent-id <id>', 'agent ID (overrides env vars and auto-construction)')
+    .option('--env <environment>', 'environment name (loads env/.env.<environment>)', 'local')
+    .option('--init-only', 'only initialize Python environment, don\'t run evaluations')
+    .option('--cache-info', 'show cache information and statistics')
+    .option('--cache-clear', 'clear the cache (removes Python runtime and venv)')
+    .option('--cache-dir', 'print the cache directory path')
+    .option('--signout', 'sign out and clear cached authentication tokens');
+  
+  program.parse(process.argv);
+  const options = program.opts();
+  
+  // Handle cache commands first (they don't need environment validation)
+  if (options.cacheInfo) {
+    console.log('🗂️  Cache Information\n');
+    const stats = await getCacheStats();
+    
+    console.log(`Cache Directory: ${stats.cacheDir}`);
+    console.log(`Cache Exists: ${stats.exists ? '✓' : '✗'}`);
+    
+    if (stats.exists) {
+      console.log(`Total Size: ${formatBytes(stats.size)}`);
+      console.log('\nComponents:');
+      console.log(`  Python Runtime: ${stats.pythonRuntime ? '✓' : '✗'}`);
+      console.log(`  Virtual Env: ${stats.venv ? '✓' : '✗'}`);
+      console.log(`  Downloads: ${stats.downloads ? '✓' : '✗'}`);
+    } else {
+      console.log('\nCache is empty. Run "runevals --init-only" to initialize.');
+    }
+    
+    process.exit(0);
+  }
+  
+  if (options.cacheClear) {
+    console.log('🗑️  Clearing cache...\n');
+    const success = await clearCache(options.verbose);
+    
+    if (success) {
+      console.log('✅ Cache cleared successfully!');
+      console.log('Run "runevals --init-only" to reinitialize.');
+    } else {
+      console.error('❌ Failed to clear cache');
+      process.exit(1);
+    }
+    
+    process.exit(0);
+  }
+  
+  if (options.cacheDir) {
+    console.log(getCacheDir());
+    process.exit(0);
+  }
+  
+  // Load environment file if specified
+  const envVars = {};
+  let resolvedAgentId = options.agentId;
+  
+  // Check for .env.local in current directory (ATK projects)
+  // First check for .env.local directly in current directory
+  let localEnvPath = path.join(process.cwd(), '.env.local');
+  let localEnvFound = false;
+  
+  if (fs.existsSync(localEnvPath)) {
+    if (!options.quiet && options.verbose) {
+      console.log(`📂 Loading .env.local from current directory`);
+    }
+    const localEnvVars = loadEnvFile(localEnvPath) || {};
+    Object.assign(envVars, localEnvVars);
+    localEnvFound = true;
+  }
+  
+  // If not found, check for env.local in env subfolder of current directory
+  if (!localEnvFound) {
+    localEnvPath = path.join(process.cwd(), 'env', 'env.local');
+    if (fs.existsSync(localEnvPath)) {
+      if (!options.quiet && options.verbose) {
+        console.log(`📂 Loading env.local from current directory env folder`);
+      }
+      const localEnvVars = loadEnvFile(localEnvPath) || {};
+      Object.assign(envVars, localEnvVars);
+      localEnvFound = true;
+    }
+  }
+  
+  if (options.env) {
+    // First check current directory's env folder
+    let envFilePath = path.join(process.cwd(), 'env', `.env.${options.env}`);
+    let envFileFound = false;
+    
+    if (fs.existsSync(envFilePath)) {
+      if (!options.quiet) {
+        console.log(`📂 Loading environment: ${options.env} from current directory env folder`);
+      }
+      const fileEnvVars = loadEnvFile(envFilePath) || {};
+      Object.assign(envVars, fileEnvVars);
+      envFileFound = true;
+    } else {
+      // Fallback to package's env directory
+      envFilePath = path.join(__dirname, '..', 'env', `.env.${options.env}`);
+      
+      if (fs.existsSync(envFilePath)) {
+        if (!options.quiet) {
+          console.log(`📂 Loading environment: ${options.env} from package env folder`);
+        }
+        const fileEnvVars = loadEnvFile(envFilePath) || {};
+        Object.assign(envVars, fileEnvVars);
+        envFileFound = true;
+      }
+    }
+    
+    if (envFileFound) {
+      // Auto-construct agent ID if not explicitly provided
+      if (!resolvedAgentId) {
+        resolvedAgentId = constructAgentId(envVars);
+        if (resolvedAgentId && !options.quiet) {
+          console.log(`🤖 Agent ID (from M365_TITLE_ID): ${resolvedAgentId}`);
+        }
+      }
+    } else if (options.env !== 'dev') {
+      // Only warn if non-default env specified
+      console.warn(`⚠️  Environment file not found: .env.${options.env}`);
+      console.warn(`   Searched in: ${path.join(process.cwd(), 'env')} and ${path.join(__dirname, '..', 'env')}`);
+      console.warn(`   Continuing with system environment variables...\n`);
+    }
+  }
+  
+  // Auto-construct agent ID from loaded env vars if not explicitly provided
+  if (!resolvedAgentId && Object.keys(envVars).length > 0) {
+    resolvedAgentId = constructAgentId(envVars);
+    if (resolvedAgentId && !options.quiet) {
+      console.log(`🤖 Agent ID (from M365_TITLE_ID): ${resolvedAgentId}`);
+    }
+  }
+  
+  // Fallback to M365_AGENT_ID env var if still not resolved
+  if (!resolvedAgentId) {
+    resolvedAgentId = process.env.M365_AGENT_ID;
+  }
+  
+  // Validate required environment variables (skip for init-only and cache commands)
+  if (!options.initOnly && !options.quiet) {
+    if (!validateEnvironmentVariables(options.env, envVars)) {
+      process.exit(1);
+    }
+  }
+  
+  // Initialize Python environment
+  console.log('🚀 M365 Copilot Agent Evaluations CLI\n');
+  await initializePythonEnvironment(options.verbose);
+  
+  // If --init-only, stop here
+  if (options.initOnly) {
+    console.log('\n✅ Python environment initialized successfully!\n');
+    console.log('⚠️  Note: Configure environment variables before running evaluations.');
+    console.log('📖 See README.md for complete setup guide.\n');
+    return;
+  }
+  
+  // Build arguments to pass to Python CLI
+  const pythonArgs = [];
+  
+  if (options.verbose) pythonArgs.push('--verbose');
+  if (options.quiet) pythonArgs.push('--quiet');
+  if (options.interactive) pythonArgs.push('--interactive');
+  if (resolvedAgentId) pythonArgs.push('--agent-id', resolvedAgentId);
+
+  // Handle signout
+  if (options.signout) {
+    console.log('🔓 Signing out...\n');
+    
+    try {      
+      // Execute Python CLI with --signout flag
+      await executePythonCli(MAIN_SCRIPT, [...pythonArgs, '--signout'], { cwd: PYTHON_CLI_DIR });
+      
+      console.log('✅ Successfully signed out and cleared cached authentication tokens!');
+    } catch (error) {
+      console.error('❌ Failed to sign out:', error.message);
+      if (options.verbose) {
+        console.error('\nFull error:', error);
+      }
+      process.exit(1);
+    }
+    
+    process.exit(0);
+  }
+  
+  // Default prompts file lookup in current working directory
+  let promptsFile = options.promptsFile;
+  if (!promptsFile && !options.prompts && !options.interactive) {
+    // Look for common prompts file names in current directory and ./evals/ subdirectory
+    const searchLocations = [
+      { dir: process.cwd(), files: ['prompts.json', 'evals.json', 'tests.json'] },
+      { dir: path.join(process.cwd(), 'evals'), files: ['prompts.json', 'evals.json', 'tests.json'] }
+    ];
+    
+    for (const location of searchLocations) {
+      for (const filename of location.files) {
+        const candidatePath = path.join(location.dir, filename);
+        if (fs.existsSync(candidatePath)) {
+          promptsFile = candidatePath;
+          if (!options.quiet) {
+            const displayPath = candidatePath.startsWith(process.cwd()) 
+              ? './' + path.relative(process.cwd(), candidatePath)
+              : candidatePath;
+            console.log(`📄 Using prompts file: ${displayPath}`);
+          }
+          break;
+        }
+      }
+      if (promptsFile) break;
+    }
+    
+    // If no prompts file found, offer to create starter file
+    if (!promptsFile && !options.quiet) {
+      console.log('\n⚠️  No prompts file found in current directory or ./evals/\n');
+      
+      // Use readline for interactive prompt
+      const readline = await import('readline');
+      const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+      });
+      
+      const answer = await new Promise((resolve) => {
+        rl.question('Create a starter evals file with sample prompts? (Y/n): ', resolve);
+      });
+      rl.close();
+      
+      const response = answer.trim().toLowerCase();
+      if (response === '' || response === 'y' || response === 'yes') {
+        const sourcePath = path.join(PYTHON_CLI_DIR, 'samples', 'starter.json');
+        
+        // Create ./evals/ directory if it doesn't exist
+        const evalsDir = path.join(process.cwd(), 'evals');
+        if (!fs.existsSync(evalsDir)) {
+          fs.mkdirSync(evalsDir, { recursive: true });
+        }
+        
+        const targetPath = path.join(evalsDir, 'evals.json');
+        fs.copyFileSync(sourcePath, targetPath);
+        
+        console.log('\n✅ Created ./evals/evals.json with 2 starter prompts');
+        console.log('Edit this file with your own prompts, then run "npx runevals" again.\n');
+        process.exit(0);
+      } else {
+        console.log('\nUse --prompts or --interactive to continue without a file:\n');
+        console.log('  npx runevals --prompts "Your prompt here"');
+        console.log('  npx runevals --interactive');
+        console.log('  npx runevals --prompts-file path/to/prompts.json\n');
+        process.exit(0);
+      }
+    }
+  }
+  
+  // Handle output file with automatic .evals directory and timestamping
+  let outputFile = options.output;
+  if (outputFile) {
+    pythonArgs.push('--output', outputFile);
+  } else if (promptsFile) {
+    // Auto-generate timestamped output in .evals directory
+    const evalsDir = path.join(process.cwd(), '.evals');
+    if (!fs.existsSync(evalsDir)) {
+      fs.mkdirSync(evalsDir, { recursive: true });
+      if (options.verbose) {
+        console.log(`Created .evals directory`);
+      }
+    }
+    
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').replace('T', '_').split('Z')[0];
+    outputFile = path.join(evalsDir, `${timestamp}.html`);
+    pythonArgs.push('--output', outputFile);
+  }
+  
+  if (promptsFile) pythonArgs.push('--prompts-file', promptsFile);
+  
+  if (options.prompts && options.prompts.length > 0) {
+    pythonArgs.push('--prompts', ...options.prompts);
+  }
+  
+  if (options.expected && options.expected.length > 0) {
+    pythonArgs.push('--expected', ...options.expected);
+  }
+  
+  // Execute Python CLI with working directory set to Python CLI directory
+  if (!options.quiet) {
+    console.log('\n📊 Running evaluations...\n');
+    console.log('─────────────────────────────────────────────────────────────\n');
+  }
+  
+  try {
+    await executePythonCli(MAIN_SCRIPT, pythonArgs, { cwd: PYTHON_CLI_DIR });
+    
+    if (!options.quiet) {
+      console.log('\n─────────────────────────────────────────────────────────────\n');
+      console.log('✓ Evals completed successfully!');
+      if (outputFile) {
+        console.log(`\nResults saved to: ${outputFile}\n`);
+      }
+    }
+  } catch (error) {
+    console.error('\n─────────────────────────────────────────────────────────────\n');
+    console.error('❌ Evaluation failed:', error.message);
+    process.exit(1);
+  }
+}
+
+// Run the CLI
+main().catch((error) => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});

package/src/clients/node-js/config/default.js

@@ -0,0 +1,25 @@
+/**
+ * Build-time injected default values
+ * DO NOT EDIT - This file is auto-generated during build.
+ *
+ * Generated: 2026-01-21T21:10:04.863Z
+ *
+ * @copyright Microsoft Corporation. All rights reserved.
+ * @license MIT
+ */
+
+export default {
+  copilotApi: {
+    /** Microsoft M365 Evaluation Client ID  */
+    m365EvalClientId: "c678803a-d8e9-4d67-849c-3a8b2d7ba5d3",
+
+    /** Copilot OAuth Scopes */
+    copilotScopes: "https://substrate.office.com/sydney/.default",
+
+    /** Copilot API Endpoint */
+    copilotApiEndpoint: "https://substrate.office.com/m365Copilot",
+
+    /** Scenario Header for Copilot API */
+    scenarioHeader: "agenticevaluation"
+  }
+};

package/src/clients/node-js/lib/cache-utils.js

@@ -0,0 +1,119 @@
+import { getCacheDir } from './python-runtime.js';
+import fs from 'fs/promises';
+
+/**
+ * Get cache statistics
+ */
+export async function getCacheStats() {
+  const cacheDir = getCacheDir();
+  const stats = {
+    cacheDir,
+    exists: false,
+    size: 0,
+    pythonRuntime: false,
+    venv: false,
+    downloads: false,
+  };
+
+  try {
+    await fs.access(cacheDir);
+    stats.exists = true;
+
+    // Check for Python runtime
+    try {
+      const pythonDir = `${cacheDir}/python`;
+      await fs.access(pythonDir);
+      stats.pythonRuntime = true;
+    } catch {
+      // Not found
+    }
+
+    // Check for venv
+    try {
+      const venvDir = `${cacheDir}/venv`;
+      await fs.access(venvDir);
+      stats.venv = true;
+    } catch {
+      // Not found
+    }
+
+    // Check for downloads
+    try {
+      const downloadsDir = `${cacheDir}/downloads`;
+      await fs.access(downloadsDir);
+      stats.downloads = true;
+    } catch {
+      // Not found
+    }
+
+    // Calculate total size (approximate)
+    stats.size = await getDirectorySize(cacheDir);
+  } catch {
+    // Cache doesn't exist
+  }
+
+  return stats;
+}
+
+/**
+ * Get directory size recursively
+ */
+async function getDirectorySize(dirPath) {
+  let totalSize = 0;
+
+  try {
+    const items = await fs.readdir(dirPath, { withFileTypes: true });
+
+    for (const item of items) {
+      const fullPath = `${dirPath}/${item.name}`;
+
+      if (item.isDirectory()) {
+        totalSize += await getDirectorySize(fullPath);
+      } else if (item.isFile()) {
+        const stats = await fs.stat(fullPath);
+        totalSize += stats.size;
+      }
+    }
+  } catch {
+    // Ignore errors (permission denied, etc.)
+  }
+
+  return totalSize;
+}
+
+/**
+ * Clear cache
+ */
+export async function clearCache(verbose = false) {
+  const cacheDir = getCacheDir();
+
+  if (verbose) {
+    console.log(`Clearing cache: ${cacheDir}`);
+  }
+
+  try {
+    await fs.rm(cacheDir, { recursive: true, force: true });
+    if (verbose) {
+      console.log('Cache cleared successfully');
+    }
+    return true;
+  } catch (err) {
+    if (verbose) {
+      console.error('Error clearing cache:', err.message);
+    }
+    return false;
+  }
+}
+
+/**
+ * Format bytes to human-readable string
+ */
+export function formatBytes(bytes) {
+  if (bytes === 0) return '0 B';
+
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
+}