@microsoft/m365-copilot-eval 1.0.1-preview.1

package/src/clients/node-js/lib/expiry-check.js

@@ -0,0 +1,164 @@
+/**
+ * Package expiry check utility
+ * Checks if the package has expired based on publish date + 60 days
+ */
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+import fs from 'fs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const EXPIRY_DAYS = 60;
+const EXPIRY_MESSAGE =
+  'This version of the M365 Evals CLI has stopped working and must be updated';
+
+/**
+ * Get the package publish date from package.json
+ * @returns {Date|null} The publish date or null if not found
+ */
+function getPackagePublishDate() {
+  try {
+    const packageJsonPath = path.join(
+      __dirname,
+      '..',
+      '..',
+      '..',
+      '..',
+      'package.json'
+    );
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+
+    // Check if publishDate is set in package.json
+    if (packageJson.publishDate) {
+      return new Date(packageJson.publishDate);
+    }
+
+    // Fall back to package.json modification time as an approximation
+    const stats = fs.statSync(packageJsonPath);
+    return stats.mtime;
+  } catch (error) {
+    console.warn('Warning: Could not determine package publish date');
+    return error;
+  }
+}
+
+/**
+ * Calculate the expiry date (publish date + 60 days)
+ * @param {Date} publishDate - The package publish date
+ * @returns {Date} The expiry date
+ */
+function calculateExpiryDate(publishDate) {
+  const expiryDate = new Date(publishDate);
+  expiryDate.setDate(expiryDate.getDate() + EXPIRY_DAYS);
+  return expiryDate;
+}
+
+/**
+ * Check if the package has expired
+ * @returns {boolean} True if expired, false otherwise
+ */
+export function isPackageExpired() {
+  const publishDate = getPackagePublishDate();
+
+  if (!publishDate) {
+    // If we can't determine the publish date, don't block execution
+    return false;
+  }
+
+  const expiryDate = calculateExpiryDate(publishDate);
+  const now = new Date();
+
+  return now > expiryDate;
+}
+
+/**
+ * Get information about the package expiry status
+ * @returns {Object} Expiry information including dates and days remaining
+ */
+export function getExpiryInfo() {
+  const publishDate = getPackagePublishDate();
+
+  if (!publishDate) {
+    return {
+      expired: false,
+      publishDate: null,
+      expiryDate: null,
+      daysRemaining: null,
+    };
+  }
+
+  const expiryDate = calculateExpiryDate(publishDate);
+  const now = new Date();
+  const daysRemaining = Math.ceil((expiryDate - now) / (1000 * 60 * 60 * 24));
+
+  return {
+    expired: now > expiryDate,
+    publishDate: publishDate.toISOString().split('T')[0],
+    expiryDate: expiryDate.toISOString().split('T')[0],
+    daysRemaining: daysRemaining > 0 ? daysRemaining : 0,
+  };
+}
+
+/**
+ * Calculate expiry information for a given publish date
+ * @param {string|Date|null} publishDate - The publish date
+ * @returns {Object} Expiry information
+ */
+export function calculateExpiryInfo(publishDate) {
+  if (!publishDate) {
+    return {
+      expired: false,
+      publishDate: null,
+      expiryDate: null,
+      daysRemaining: null,
+    };
+  }
+
+  const pubDate =
+    typeof publishDate === 'string' ? new Date(publishDate) : publishDate;
+  const expiryDate = new Date(pubDate);
+  expiryDate.setDate(expiryDate.getDate() + EXPIRY_DAYS);
+
+  const now = new Date();
+  const daysRemaining = Math.ceil((expiryDate - now) / (1000 * 60 * 60 * 24));
+
+  return {
+    expired: now >= expiryDate,
+    publishDate: pubDate.toISOString().split('T')[0],
+    expiryDate: expiryDate.toISOString().split('T')[0],
+    daysRemaining: daysRemaining > 0 ? daysRemaining : 0,
+  };
+}
+
+/**
+ * Check package expiry based on expiry info and exit or warn accordingly
+ * @param {Object} info - Expiry information from calculateExpiryInfo()
+ */
+export function handleExpiryCheck(info) {
+  // If expired, show error and exit
+  if (info.expired) {
+    console.error('\n❌ ' + EXPIRY_MESSAGE);
+    process.exit(1);
+  }
+
+  // If close to expiry (within 7 days), show warning
+  if (info.daysRemaining !== null && info.daysRemaining <= 7) {
+    console.warn(
+      '\n⚠️  This version of the M365 Evals CLI will stop working soon and should be updated.'
+    );
+    console.warn(`   Days remaining: ${info.daysRemaining}`);
+  }
+}
+
+/**
+ * Check package expiry and exit if expired, or warn if close to expiry
+ * This is the main function that should be called at startup
+ * Displays appropriate message and exits with code 1 if expired
+ */
+export function checkPackageExpiry() {
+  const publishDate = getPackagePublishDate();
+  const info = calculateExpiryInfo(publishDate);
+  handleExpiryCheck(info);
+}

package/src/clients/node-js/lib/index.js

@@ -0,0 +1,25 @@
+/**
+ * @microsoft/m365-copilot-eval
+ *
+ * Zero-config Node.js wrapper for M365 Copilot Agent Evaluations
+ * using Python Build Standalone and Azure AI Evaluation SDK
+ */
+
+export {
+  ensurePythonRuntime,
+  getPythonExecutable,
+  getCacheDir,
+  getPlatformKey,
+} from './python-runtime.js';
+
+export { ensureVenv, executePythonCli, getVenvDir } from './venv-manager.js';
+
+export { getCacheStats, clearCache, formatBytes } from './cache-utils.js';
+
+export {
+  checkPackageExpiry,
+  calculateExpiryInfo,
+  handleExpiryCheck,
+  isPackageExpired,
+  getExpiryInfo,
+} from './expiry-check.js';

package/src/clients/node-js/lib/python-runtime.js

@@ -0,0 +1,253 @@
+import os from 'os';
+import path from 'path';
+import fs from 'fs/promises';
+import crypto from 'crypto';
+import { pipeline } from 'stream/promises';
+import { createWriteStream } from 'fs';
+import fetch from 'node-fetch';
+
+/**
+ * Python Build Standalone (PBS) configuration
+ * Using Python 3.13.10 builds from https://github.com/indygreg/python-build-standalone
+ */
+
+// PBS release version (latest release with Python 3.13.10)
+const PBS_VERSION = '20251202';
+const PYTHON_VERSION = '3.13.10';
+
+// Base URL for PBS releases
+const PBS_BASE_URL = `https://github.com/indygreg/python-build-standalone/releases/download/${PBS_VERSION}`;
+
+/**
+ * Platform-specific PBS distribution mapping
+ * Format: cpython-{version}+{build}-{arch}-{vendor}-{os}-{profile}.tar.{compression}
+ */
+const PBS_DISTRIBUTIONS = {
+  'darwin-x64': {
+    filename: `cpython-${PYTHON_VERSION}+${PBS_VERSION}-x86_64-apple-darwin-install_only.tar.gz`,
+    sha256: 'a02761a4f189f71c0512e88df7ca2843696d61da659e47f8a5c8a9bd2c0d16f4', // Placeholder - needs actual hash
+  },
+  'darwin-arm64': {
+    filename: `cpython-${PYTHON_VERSION}+${PBS_VERSION}-aarch64-apple-darwin-install_only.tar.gz`,
+    sha256: '37afe4e77ab62ac50f197b1cb1f3bc02c82735c6be893da0996afcde5dc41048', // Placeholder
+  },
+  'linux-x64': {
+    filename: `cpython-${PYTHON_VERSION}+${PBS_VERSION}-x86_64-unknown-linux-gnu-install_only.tar.gz`,
+    sha256: '0cac1495fff920219904b1d573aaec0df54d549c226cb45f5c60cb6d2c72727a', // Placeholder
+  },
+  'linux-arm64': {
+    filename: `cpython-${PYTHON_VERSION}+${PBS_VERSION}-aarch64-unknown-linux-gnu-install_only.tar.gz`,
+    sha256: 'c68280591cda1c9515a04809fa6926020177e8e5892300206e0496ea1d10290e', // Placeholder
+  },
+  'win32-x64': {
+    filename: `cpython-${PYTHON_VERSION}+${PBS_VERSION}-x86_64-pc-windows-msvc-install_only.tar.gz`,
+    sha256: '8b00014c7c35f9ad4cb1c565f067500bacc4125c8bc30e4389ee0be9fd6ffa3d',
+  },
+  'win32-arm64': {
+    filename: `cpython-${PYTHON_VERSION}+${PBS_VERSION}-aarch64-pc-windows-msvc-install_only.tar.gz`,
+    sha256: '9060d644bd32ac0e0af970d0b21e207e6ff416b7c4dc26ffc4f9b043fb45b463',
+  },
+};
+
+/**
+ * Get the platform key for the current system
+ */
+export function getPlatformKey() {
+  const platform = process.platform;
+  const arch = process.arch;
+
+  const platformKey = `${platform}-${arch}`;
+
+  if (!PBS_DISTRIBUTIONS[platformKey]) {
+    throw new Error(
+      `Unsupported platform: ${platform}-${arch}. ` +
+        `Supported platforms: ${Object.keys(PBS_DISTRIBUTIONS).join(', ')}`
+    );
+  }
+
+  return platformKey;
+}
+
+/**
+ * Get cache directory for Python runtime
+ * Priority: RUNEVALS_CACHE_DIR > XDG/platform cache > temp
+ */
+export function getCacheDir() {
+  if (process.env.RUNEVALS_CACHE_DIR) {
+    return path.resolve(process.env.RUNEVALS_CACHE_DIR);
+  }
+
+  const homeDir = os.homedir();
+
+  switch (process.platform) {
+    case 'darwin':
+      return path.join(homeDir, 'Library', 'Caches', 'm365-copilot-eval');
+    case 'win32':
+      return path.join(
+        process.env.LOCALAPPDATA || path.join(homeDir, 'AppData', 'Local'),
+        'm365-copilot-eval'
+      );
+    default: // Linux and others
+      return path.join(
+        process.env.XDG_CACHE_HOME || path.join(homeDir, '.cache'),
+        'm365-copilot-eval'
+      );
+  }
+}
+
+/**
+ * Calculate SHA256 hash of a file
+ */
+async function calculateFileHash(filePath) {
+  const hash = crypto.createHash('sha256');
+  const stream = (await import('fs')).createReadStream(filePath);
+
+  for await (const chunk of stream) {
+    hash.update(chunk);
+  }
+
+  return hash.digest('hex');
+}
+
+/**
+ * Download a file with progress indication
+ */
+async function downloadFile(url, destPath, expectedHash) {
+  console.log(`Downloading: ${url}`);
+
+  // Support proxy configuration
+  const fetchOptions = {};
+  if (process.env.HTTP_PROXY || process.env.HTTPS_PROXY) {
+    const { HttpsProxyAgent } = await import('https-proxy-agent');
+    const proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
+    fetchOptions.agent = new HttpsProxyAgent(proxy);
+  }
+
+  const response = await fetch(url, fetchOptions);
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to download: ${response.status} ${response.statusText}`
+    );
+  }
+
+  // Create parent directory if needed
+  await fs.mkdir(path.dirname(destPath), { recursive: true });
+
+  // Stream to file
+  const fileStream = createWriteStream(destPath);
+  await pipeline(response.body, fileStream);
+
+  // Verify checksum
+  console.log('Verifying checksum...');
+  const actualHash = await calculateFileHash(destPath);
+
+  if (actualHash !== expectedHash) {
+    await fs.unlink(destPath); // Remove corrupted file
+    throw new Error(
+      `Checksum mismatch!\n` +
+        `  Expected: ${expectedHash}\n` +
+        `  Actual:   ${actualHash}\n` +
+        `This could indicate a corrupted download or a supply chain attack.`
+    );
+  }
+
+  console.log('Checksum verified ✓');
+}
+
+/**
+ * Extract tar.gz archive
+ */
+async function extractTarGz(archivePath, destDir) {
+  const tar = await import('tar');
+
+  console.log(`Extracting to: ${destDir}`);
+  await fs.mkdir(destDir, { recursive: true });
+
+  await tar.extract({
+    file: archivePath,
+    cwd: destDir,
+    strip: 1, // Remove the top-level directory from archive
+  });
+
+  console.log('Extraction complete ✓');
+}
+
+/**
+ * Download and setup Python Build Standalone runtime
+ * Returns the path to the Python executable
+ */
+export async function ensurePythonRuntime(verbose = false) {
+  const platformKey = getPlatformKey();
+  const distribution = PBS_DISTRIBUTIONS[platformKey];
+
+  const cacheDir = getCacheDir();
+  const pythonDir = path.join(
+    cacheDir,
+    'python',
+    `${PYTHON_VERSION}-${platformKey}`
+  );
+  const archivePath = path.join(cacheDir, 'downloads', distribution.filename);
+
+  // Determine Python executable path based on platform
+  let pythonExe;
+  if (process.platform === 'win32') {
+    pythonExe = path.join(pythonDir, 'python.exe');
+  } else {
+    pythonExe = path.join(pythonDir, 'bin', 'python3');
+  }
+
+  // Check if Python is already installed
+  try {
+    await fs.access(pythonExe);
+    if (verbose) {
+      console.log(`Using cached Python runtime: ${pythonExe}`);
+    }
+    return pythonExe;
+  } catch {
+    // Python not found, proceed with download and extraction
+  }
+
+  console.log(
+    `Setting up Python ${PYTHON_VERSION} runtime for ${platformKey}...`
+  );
+
+  // Download if not cached
+  try {
+    await fs.access(archivePath);
+    if (verbose) {
+      console.log('Using cached download');
+    }
+  } catch {
+    const downloadUrl = `${PBS_BASE_URL}/${distribution.filename}`;
+    await downloadFile(downloadUrl, archivePath, distribution.sha256);
+  }
+
+  // Extract
+  await extractTarGz(archivePath, pythonDir);
+
+  // Verify Python executable exists
+  await fs.access(pythonExe);
+
+  console.log(`Python runtime ready: ${pythonExe}`);
+  return pythonExe;
+}
+
+/**
+ * Get Python executable path (assumes runtime is already set up)
+ */
+export async function getPythonExecutable() {
+  const platformKey = getPlatformKey();
+  const cacheDir = getCacheDir();
+  const pythonDir = path.join(
+    cacheDir,
+    'python',
+    `${PYTHON_VERSION}-${platformKey}`
+  );
+
+  if (process.platform === 'win32') {
+    return path.join(pythonDir, 'python.exe');
+  } else {
+    return path.join(pythonDir, 'bin', 'python3');
+  }
+}

package/src/clients/node-js/lib/venv-manager.js

@@ -0,0 +1,242 @@
+import path from 'path';
+import fs from 'fs/promises';
+import { spawn } from 'child_process';
+import crypto from 'crypto';
+import { getPythonExecutable, getCacheDir } from './python-runtime.js';
+
+/**
+ * Execute a command and return stdout
+ */
+function execCommand(command, args, options = {}) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(command, args, {
+      ...options,
+      shell: process.platform === 'win32',
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    if (proc.stdout) {
+      proc.stdout.on('data', (data) => {
+        stdout += data.toString();
+        if (options.verbose) {
+          process.stdout.write(data);
+        }
+      });
+    }
+
+    if (proc.stderr) {
+      proc.stderr.on('data', (data) => {
+        stderr += data.toString();
+        if (options.verbose) {
+          process.stderr.write(data);
+        }
+      });
+    }
+
+    proc.on('error', reject);
+
+    proc.on('close', (code) => {
+      if (code === 0) {
+        resolve({ stdout, stderr, code });
+      } else {
+        reject(new Error(`Command failed with code ${code}\n${stderr}`));
+      }
+    });
+  });
+}
+
+/**
+ * Calculate hash of requirements file for cache invalidation
+ */
+async function getRequirementsHash(requirementsPath) {
+  const content = await fs.readFile(requirementsPath, 'utf-8');
+  return crypto
+    .createHash('sha256')
+    .update(content)
+    .digest('hex')
+    .substring(0, 16);
+}
+
+/**
+ * Get the venv directory path
+ */
+export function getVenvDir() {
+  const cacheDir = getCacheDir();
+  return path.join(cacheDir, 'venv');
+}
+
+/**
+ * Get the venv Python executable path
+ */
+function getVenvPython(venvDir) {
+  if (process.platform === 'win32') {
+    return path.join(venvDir, 'Scripts', 'python.exe');
+  } else {
+    return path.join(venvDir, 'bin', 'python');
+  }
+}
+
+/**
+ * Get the venv pip executable path
+ */
+function getVenvPip(venvDir) {
+  if (process.platform === 'win32') {
+    return path.join(venvDir, 'Scripts', 'pip.exe');
+  } else {
+    return path.join(venvDir, 'bin', 'pip');
+  }
+}
+
+/**
+ * Create a virtual environment
+ */
+async function createVenv(pythonExe, venvDir, verbose = false) {
+  if (verbose) {
+    console.log(`Creating virtual environment at: ${venvDir}`);
+  }
+
+  await fs.mkdir(venvDir, { recursive: true });
+
+  await execCommand(pythonExe, ['-m', 'venv', venvDir], { verbose });
+
+  if (verbose) {
+    console.log('Virtual environment created ✓');
+  }
+}
+
+/**
+ * Install packages from requirements.txt into venv
+ */
+async function installRequirements(venvDir, requirementsPath, verbose = false) {
+  const pipExe = getVenvPip(venvDir);
+
+  console.log('Installing Python dependencies...');
+  if (verbose) {
+    console.log(`Using pip: ${pipExe}`);
+    console.log(`Requirements: ${requirementsPath}`);
+  }
+
+  // Upgrade pip first
+  await execCommand(pipExe, ['install', '--upgrade', 'pip'], { verbose });
+
+  // Install requirements with hash checking if available
+  const args = ['install', '-r', requirementsPath];
+
+  // Support proxy and certificate configuration
+  if (process.env.PIP_CERT) {
+    args.push('--cert', process.env.PIP_CERT);
+  }
+
+  if (process.env.PIP_TRUSTED_HOST) {
+    args.push('--trusted-host', process.env.PIP_TRUSTED_HOST);
+  }
+
+  await execCommand(pipExe, args, { verbose });
+
+  console.log('Dependencies installed ✓');
+}
+
+/**
+ * Check if venv exists and is valid
+ */
+async function isVenvValid(venvDir, requirementsHash) {
+  try {
+    // Check if venv Python exists
+    const venvPython = getVenvPython(venvDir);
+    await fs.access(venvPython);
+
+    // Check if requirements hash matches
+    const markerFile = path.join(venvDir, '.requirements-hash');
+    try {
+      const storedHash = await fs.readFile(markerFile, 'utf-8');
+      return storedHash.trim() === requirementsHash;
+    } catch {
+      return false; // Marker file doesn't exist
+    }
+  } catch {
+    return false; // Venv doesn't exist
+  }
+}
+
+/**
+ * Write requirements hash marker
+ */
+async function writeRequirementsMarker(venvDir, requirementsHash) {
+  const markerFile = path.join(venvDir, '.requirements-hash');
+  await fs.writeFile(markerFile, requirementsHash, 'utf-8');
+}
+
+/**
+ * Ensure virtual environment is set up with all dependencies
+ * Returns the path to the venv Python executable
+ */
+export async function ensureVenv(requirementsPath, verbose = false) {
+  const pythonExe = await getPythonExecutable();
+  const venvDir = getVenvDir();
+  const requirementsHash = await getRequirementsHash(requirementsPath);
+
+  // Check if venv is already set up and valid
+  if (await isVenvValid(venvDir, requirementsHash)) {
+    if (verbose) {
+      console.log('Using existing virtual environment');
+    }
+    return getVenvPython(venvDir);
+  }
+
+  console.log('Setting up Python virtual environment...');
+
+  // Remove old venv if it exists but is invalid
+  try {
+    await fs.rm(venvDir, { recursive: true, force: true });
+  } catch {
+    // Ignore errors during cleanup
+  }
+
+  // Create new venv
+  await createVenv(pythonExe, venvDir, verbose);
+
+  // Install requirements
+  await installRequirements(venvDir, requirementsPath, verbose);
+
+  // Mark as complete
+  await writeRequirementsMarker(venvDir, requirementsHash);
+
+  console.log('Virtual environment ready ✓');
+  return getVenvPython(venvDir);
+}
+
+/**
+ * Execute Python CLI script with the venv Python
+ */
+export async function executePythonCli(scriptPath, args = [], options = {}) {
+  const venvPython = getVenvPython(getVenvDir());
+
+  if (!options.quiet) {
+    console.log(`Executing Python CLI: ${scriptPath} ${args.join(' ')}`);
+  }
+
+  return new Promise((resolve, reject) => {
+    const proc = spawn(venvPython, [scriptPath, ...args], {
+      stdio: 'inherit', // Pass through stdin/stdout/stderr for interactive auth
+      env: {
+        ...process.env,
+        // Ensure venv is activated by setting PATH
+        PYTHONHOME: '', // Clear PYTHONHOME to avoid conflicts
+      },
+      cwd: options.cwd || process.cwd(), // Allow custom working directory
+      ...options,
+    });
+
+    proc.on('error', reject);
+
+    proc.on('close', (code) => {
+      if (code === 0) {
+        resolve(code);
+      } else {
+        reject(new Error(`Python CLI exited with code ${code}`));
+      }
+    });
+  });
+}