@microsoft/m365-copilot-eval 1.4.0-preview.1 → 1.6.0-preview.1

package/src/clients/node-js/lib/python-runtime.js

@@ -5,6 +5,10 @@ import crypto from 'crypto';
 import { pipeline } from 'stream/promises';
 import { createWriteStream } from 'fs';
 import { Transform } from 'stream';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+
+const execFileAsync = promisify(execFile);
 import fetch from 'node-fetch';
 
 /**
@@ -14,7 +18,7 @@ import fetch from 'node-fetch';
 
 // PBS release version (latest release with Python 3.13.10)
 const PBS_VERSION = '20251202';
-const PYTHON_VERSION = '3.13.10';
+export const PYTHON_VERSION = '3.13.10';
 
 // Base URL for PBS releases
 const PBS_BASE_URL = `https://github.com/indygreg/python-build-standalone/releases/download/${PBS_VERSION}`;
@@ -212,66 +216,111 @@ async function extractTarGz(archivePath, destDir, onProgress) {
 }
 
 /**
- * Download and setup Python Build Standalone runtime
- * Returns the path to the Python executable
- * @param {boolean} [verbose=false] - Enable verbose output
- * @param {Function} [onProgress] - Optional progress callback
+ * Validate that a Python executable is the expected version.
+ * @param {string} pythonPath - Path to a Python executable
+ * @returns {{ isMatch: boolean, version: string }} Version info
  */
-export async function ensurePythonRuntime(verbose = false, onProgress) {
-  const platformKey = getPlatformKey();
-  const distribution = PBS_DISTRIBUTIONS[platformKey];
-
-  const cacheDir = getCacheDir();
-  const pythonDir = path.join(
-    cacheDir,
-    'python',
-    `${PYTHON_VERSION}-${platformKey}`
-  );
-  const archivePath = path.join(cacheDir, 'downloads', distribution.filename);
-
-  // Determine Python executable path based on platform
-  const pythonExe =
-    process.platform === 'win32'
-      ? path.join(pythonDir, 'python.exe')
-      : path.join(pythonDir, 'bin', 'python3');
-
-  // Check if Python is already installed (silent skip per FR-007)
+export async function _validatePythonVersion(pythonPath) {
+  const expectedMajorMinor = PYTHON_VERSION.split('.').slice(0, 2).join('.');
   try {
-    await fs.access(pythonExe);
-    if (verbose) {
-      console.log(`Using cached Python runtime: ${pythonExe}`);
+    const { stdout } = await execFileAsync(pythonPath, ['--version'], {
+      encoding: 'utf8',
+      timeout: 10_000,
+    });
+    // Output format: "Python 3.13.10"
+    const match = stdout.trim().match(/Python\s+(\d+\.\d+\.\d+)/);
+    if (!match) {
+      return { isMatch: false, version: 'unknown' };
     }
-    // Skip both download and extract phases silently
-    onProgress?.({ type: 'skip', phaseId: 'download' });
-    onProgress?.({ type: 'skip', phaseId: 'extract' });
-    return pythonExe;
+    const version = match[1];
+    const majorMinor = version.split('.').slice(0, 2).join('.');
+    return { isMatch: majorMinor === expectedMajorMinor, version };
   } catch {
-    // Python not found, proceed with download and extraction
+    return { isMatch: false, version: 'unknown' };
   }
+}
 
-  if (!onProgress) {
-    console.log(
-      `Setting up Python ${PYTHON_VERSION} runtime for ${platformKey}...`
-    );
+/**
+ * Attempt to resolve a fallback Python from the PYTHON_PATH environment variable.
+ * @returns {Promise<{ pythonPath: string, version: string, isMatch: boolean }>}
+ */
+export async function _resolveFallbackPython() {
+  const pythonPath = process.env.PYTHON_PATH;
+  if (!pythonPath) {
+    throw new Error('PYTHON_PATH environment variable is not set.');
   }
 
-  // Download if not cached
-  let needsDownload = false;
+  await fs.access(pythonPath);
+  const { isMatch, version } = await _validatePythonVersion(pythonPath);
+  return { pythonPath, version, isMatch };
+}
+
+/**
+ * Download and setup Python Build Standalone runtime
+ * Returns the path to the Python executable and version info.
+ * On download failure, falls back to PYTHON_PATH env var if available.
+ * @param {boolean} [verbose=false] - Enable verbose output
+ * @param {Function} [onProgress] - Optional progress callback
+ * @returns {Promise<{ pythonPath: string, version: string, isMatch: boolean }>}
+ */
+export async function ensurePythonRuntime(verbose = false, onProgress) {
+  // Attempt PBS download + extraction; falls back to PYTHON_PATH on any failure
+  // (including unsupported platform where getPlatformKey() throws)
+  let currentPhase = 'download';
   try {
-    await fs.access(archivePath);
-    if (verbose) {
-      console.log('Using cached download');
+    const platformKey = getPlatformKey();
+    const distribution = PBS_DISTRIBUTIONS[platformKey];
+
+    const cacheDir = getCacheDir();
+    const pythonDir = path.join(
+      cacheDir,
+      'python',
+      `${PYTHON_VERSION}-${platformKey}`
+    );
+    const archivePath = path.join(cacheDir, 'downloads', distribution.filename);
+
+    // Determine Python executable path based on platform
+    const pythonExe =
+      process.platform === 'win32'
+        ? path.join(pythonDir, 'python.exe')
+        : path.join(pythonDir, 'bin', 'python3');
+
+    // Check if Python is already installed (silent skip per FR-007)
+    try {
+      await fs.access(pythonExe);
+      if (verbose) {
+        console.log(`Using cached Python runtime: ${pythonExe}`);
+      }
+      // Skip both download and extract phases silently
+      onProgress?.({ type: 'skip', phaseId: 'download' });
+      onProgress?.({ type: 'skip', phaseId: 'extract' });
+      return { pythonPath: pythonExe, version: PYTHON_VERSION, isMatch: true };
+    } catch {
+      // Python not found, proceed with download and extraction
     }
-    // Skip download phase silently
-    onProgress?.({ type: 'skip', phaseId: 'download' });
-  } catch {
-    needsDownload = true;
-  }
 
-  if (needsDownload) {
-    // Start download phase
-    onProgress?.({ type: 'start', phaseId: 'download' });
+    if (!onProgress) {
+      console.log(
+        `Setting up Python ${PYTHON_VERSION} runtime for ${platformKey}...`
+      );
+    }
+
+    // Download if not cached
+    let needsDownload = false;
     try {
+      await fs.access(archivePath);
+      if (verbose) {
+        console.log('Using cached download');
+      }
+      // Skip download phase silently
+      onProgress?.({ type: 'skip', phaseId: 'download' });
+    } catch {
+      needsDownload = true;
+    }
+
+    if (needsDownload) {
+      // Start download phase
+      onProgress?.({ type: 'start', phaseId: 'download' });
       const downloadUrl = `${PBS_BASE_URL}/${distribution.filename}`;
       await downloadFile(
         downloadUrl,
@@ -280,29 +329,52 @@ export async function ensurePythonRuntime(verbose = false, onProgress) {
         onProgress
       );
       onProgress?.({ type: 'complete', phaseId: 'download' });
-    } catch (error) {
-      onProgress?.({ type: 'error', phaseId: 'download', error });
-      throw error;
     }
-  }
 
-  // Extract phase
-  onProgress?.({ type: 'start', phaseId: 'extract' });
-  try {
+    // Extract phase
+    currentPhase = 'extract';
+    onProgress?.({ type: 'start', phaseId: 'extract' });
     await extractTarGz(archivePath, pythonDir, onProgress);
     onProgress?.({ type: 'complete', phaseId: 'extract' });
-  } catch (error) {
-    onProgress?.({ type: 'error', phaseId: 'extract', error });
-    throw error;
-  }
 
-  // Verify Python executable exists
-  await fs.access(pythonExe);
+    // Verify Python executable exists
+    await fs.access(pythonExe);
 
-  if (!onProgress) {
-    console.log(`Python runtime ready: ${pythonExe}`);
+    if (!onProgress) {
+      console.log(`Python runtime ready: ${pythonExe}`);
+    }
+    return { pythonPath: pythonExe, version: PYTHON_VERSION, isMatch: true };
+  } catch (pbsError) {
+    // PBS setup failed (unsupported platform, download, or extract) — attempt PYTHON_PATH fallback
+    onProgress?.({ type: 'error', phaseId: currentPhase, error: pbsError });
+
+    if (verbose) {
+      console.error(
+        `Python setup failed: ${pbsError.message}. Checking PYTHON_PATH fallback...`
+      );
+    }
+
+    try {
+      const fallback = await _resolveFallbackPython();
+      if (verbose) {
+        console.log(
+          `Using PYTHON_PATH fallback: ${fallback.pythonPath} (Python ${fallback.version})`
+        );
+      }
+      // Mark extract as skipped since we're using an external Python
+      onProgress?.({ type: 'skip', phaseId: 'extract' });
+      return fallback;
+    } catch (fallbackError) {
+      // Neither PBS nor PYTHON_PATH worked
+      const err = new Error(
+        `Python ${PYTHON_VERSION} could not be set up.\n` +
+          `  PBS setup failed: ${pbsError.message}\n` +
+          `  PYTHON_PATH fallback: ${fallbackError.message}`
+      );
+      err.code = 'PYTHON_NOT_FOUND';
+      throw err;
+    }
   }
-  return pythonExe;
 }
 
 /**

package/src/clients/node-js/lib/venv-manager.js

@@ -321,9 +321,10 @@ async function writeRequirementsMarker(venvDir, requirementsHash) {
 export async function ensureVenv(
   requirementsPath,
   verbose = false,
-  onProgress
+  onProgress,
+  pythonExePath
 ) {
-  const pythonExe = await getPythonExecutable();
+  const pythonExe = pythonExePath || (await getPythonExecutable());
   const venvDir = getVenvDir();
   const requirementsHash = await getRequirementsHash(requirementsPath);
 

package/src/clients/node-js/lib/version-check.js

@@ -0,0 +1,268 @@
+import { PYTHON_VERSION } from './python-runtime.js';
+
+/**
+ * Minimum Python major.minor that is still receiving security updates.
+ *
+ * This MUST be reviewed against the official Python release schedule
+ * (https://devguide.python.org/versions/) when bumping PYTHON_VERSION
+ * or whenever a previously supported branch reaches end-of-life.
+ *
+ * Versions older than this are hard-blocked because they no longer
+ * receive security patches and pose a supply-chain risk to users.
+ */
+export const MIN_SUPPORTED_PYTHON_MAJOR_MINOR = '3.10';
+
+/**
+ * Parse a version string into { major, minor }, or null if it cannot
+ * be parsed (e.g. the literal string 'unknown').
+ * @param {string} version
+ * @returns {{ major: number, minor: number } | null}
+ */
+function parseMajorMinor(version) {
+  const match = String(version).match(/^(\d+)\.(\d+)/);
+  if (!match) return null;
+  return { major: parseInt(match[1], 10), minor: parseInt(match[2], 10) };
+}
+
+/**
+ * Returns true when the detected Python version is older than the
+ * minimum supported major.minor and is therefore considered EOL.
+ *
+ * Returns false for unparseable versions (e.g. 'unknown') so the
+ * caller's regular version-mismatch flow can handle them.
+ *
+ * @param {string} version - Detected Python version string (e.g. "3.8.10")
+ * @returns {boolean}
+ */
+export function _isPythonVersionEol(version) {
+  const detected = parseMajorMinor(version);
+  if (!detected) return false;
+  const min = parseMajorMinor(MIN_SUPPORTED_PYTHON_MAJOR_MINOR);
+  if (detected.major < min.major) return true;
+  if (detected.major === min.major && detected.minor < min.minor) return true;
+  return false;
+}
+
+/**
+ * Build the user-facing error message for an EOL Python version that
+ * has been hard-blocked. EOL versions are blocked unconditionally
+ * (no override prompt) because they no longer receive security fixes.
+ *
+ * @param {string} version - Detected Python version string
+ * @returns {string[]} Array of lines to print
+ */
+export function _formatEolPythonError(version) {
+  return [
+    `\n❌ Python ${version} has reached end-of-life and is not supported.`,
+    '   Continuing would expose you to known security vulnerabilities,',
+    '   so this version is blocked unconditionally.\n',
+    `   Minimum supported version: Python ${MIN_SUPPORTED_PYTHON_MAJOR_MINOR}`,
+    `   Recommended version:       Python ${PYTHON_VERSION}`,
+    '   Python release schedule:   https://devguide.python.org/versions/',
+    '   Download a supported build: https://www.python.org/downloads/\n',
+  ];
+}
+
+/**
+ * Build the user-facing error message for PYTHON_NOT_FOUND failures.
+ * @param {string} platform - process.platform value
+ * @returns {string[]} Array of lines to print
+ */
+export function _formatPythonNotFoundError(platform) {
+  const lines = [
+    `\n❌ Python ${PYTHON_VERSION} is required but could not be found.\n`,
+    'The automatic download failed, and no valid fallback was detected.',
+    'To provide Python manually, set the PYTHON_PATH environment variable:\n',
+  ];
+
+  if (platform === 'win32') {
+    lines.push('  set PYTHON_PATH=C:\\path\\to\\python.exe');
+  } else {
+    lines.push('  export PYTHON_PATH=/path/to/python3');
+  }
+
+  lines.push(
+    `\nRequired version: Python ${PYTHON_VERSION}`,
+    'Download: https://www.python.org/downloads/\n'
+  );
+
+  return lines;
+}
+
+/**
+ * Evaluate what action to take when the resolved Python version doesn't match.
+ *
+ * EOL versions are hard-rejected even when the user answers "yes" — this
+ * defends against the y/n prompt being used to bypass the security gate
+ * and is checked here in addition to the upfront check in the CLI.
+ *
+ * @param {string} version - The detected Python version string
+ * @param {{ isInteractive: boolean, promptAnswer?: string }} options
+ * @returns {{ action: 'continue' | 'reject', reason: string }}
+ */
+export function _evaluateVersionMismatch(
+  version,
+  { isInteractive, promptAnswer }
+) {
+  if (_isPythonVersionEol(version)) {
+    return {
+      action: 'reject',
+      reason: `Python ${version} has reached end-of-life and cannot be used. Install Python ${MIN_SUPPORTED_PYTHON_MAJOR_MINOR} or newer (Python ${PYTHON_VERSION} recommended).`,
+    };
+  }
+
+  if (!isInteractive) {
+    return {
+      action: 'reject',
+      reason: `Python version mismatch (found ${version}). In non-interactive environments, PYTHON_PATH must point to Python ${PYTHON_VERSION}.`,
+    };
+  }
+
+  const answer = (promptAnswer || '').trim().toLowerCase();
+  if (answer === 'y' || answer === 'yes') {
+    return { action: 'continue', reason: 'User accepted version mismatch.' };
+  }
+
+  return {
+    action: 'reject',
+    reason: `Aborted. Please install Python ${PYTHON_VERSION} or set PYTHON_PATH to a compatible version.`,
+  };
+}
+
+/**
+ * Handle a Python version mismatch with side effects routed through
+ * injected callbacks so the full decision tree is unit-testable without
+ * touching real stdout/stderr, readline, or process.exit.
+ *
+ * Behaviour mirrors the inline block previously in runevals.js:
+ *   - EOL versions are hard-blocked (no prompt) regardless of interactivity.
+ *   - Interactive terminals warn, prompt, and respect the user's answer
+ *     (subject to the EOL block enforced inside _evaluateVersionMismatch).
+ *   - Non-interactive environments auto-reject.
+ *
+ * @param {object} args
+ * @param {{ version: string, isMatch: boolean }} args.runtime - Detected runtime info.
+ * @param {boolean} args.isInteractive - Whether we are in an interactive terminal.
+ * @param {() => Promise<string>} [args.promptForContinue] - Async function that
+ *   resolves with the user's y/n answer. Required when isInteractive is true.
+ * @param {(msg: string) => void} args.warn - Sink for warning messages.
+ * @param {(msg: string) => void} args.error - Sink for error messages.
+ * @returns {Promise<{ shouldExit: boolean, exitCode?: number }>}
+ *   When `shouldExit` is true the caller should `process.exit(exitCode)`.
+ *   When false, initialization may continue with the mismatched runtime.
+ */
+export async function _handlePythonVersionMismatch({
+  runtime,
+  isInteractive,
+  promptForContinue,
+  warn,
+  error,
+}) {
+  if (runtime.isMatch) {
+    return { shouldExit: false };
+  }
+
+  if (_isPythonVersionEol(runtime.version)) {
+    for (const line of _formatEolPythonError(runtime.version)) {
+      error(line);
+    }
+    return { shouldExit: true, exitCode: 1 };
+  }
+
+  if (isInteractive) {
+    warn(
+      `\n⚠️  Python version mismatch. Expected ${PYTHON_VERSION}, found ${runtime.version}.`
+    );
+
+    const answer = await promptForContinue();
+
+    const result = _evaluateVersionMismatch(runtime.version, {
+      isInteractive: true,
+      promptAnswer: answer,
+    });
+    if (result.action === 'reject') {
+      error(`\n${result.reason}`);
+      error('Download: https://www.python.org/downloads/\n');
+      return { shouldExit: true, exitCode: 1 };
+    }
+    return { shouldExit: false };
+  }
+
+  // Non-interactive (CI/CD) — auto-reject version mismatch
+  const result = _evaluateVersionMismatch(runtime.version, {
+    isInteractive: false,
+  });
+  error(`\n❌ ${result.reason}`);
+  return { shouldExit: true, exitCode: 1 };
+}
+
+/**
+ * Build the user-facing error guidance lines printed when Python
+ * environment initialization fails. Returns the lines as an array so
+ * the caller can route them to console.error and the result is testable
+ * without intercepting console.
+ *
+ * If `error.code === 'PYTHON_NOT_FOUND'`, the platform-specific
+ * PYTHON_NOT_FOUND guidance is included before the generic troubleshooting
+ * tips.
+ *
+ * @param {{ error: { code?: string }, platform: string }} args
+ * @returns {string[]}
+ */
+export function _buildInitializationFailureLines({ error, platform }) {
+  const lines = [];
+
+  if (error && error.code === 'PYTHON_NOT_FOUND') {
+    for (const line of _formatPythonNotFoundError(platform)) {
+      lines.push(line);
+    }
+  }
+
+  lines.push(
+    'Troubleshooting:',
+    '  - Check your internet connection',
+    '  - If behind a proxy, set HTTP_PROXY/HTTPS_PROXY environment variables',
+    '  - For SSL issues, set NODE_EXTRA_CA_CERTS or PIP_CERT',
+    '  - Run with --log-level debug for detailed output'
+  );
+
+  return lines;
+}
+
+/**
+ * Prompt the user to continue past a Python version mismatch.
+ *
+ * The readline factory is injected so this can be unit-tested with a
+ * fake interface. The default factory uses Node's built-in readline
+ * bound to process.stdin/process.stdout. The interface is always
+ * closed via try/finally so a Ctrl+C does not leave the event loop
+ * holding stdin open.
+ *
+ * @param {object} [deps]
+ * @param {() => { question: (q: string, cb: (answer: string) => void) => void, close: () => void }} [deps.createInterface]
+ *   Factory returning a readline-compatible interface.
+ * @returns {Promise<string>} The user's raw answer (untrimmed).
+ */
+export async function _promptForContinueWithMismatch(deps = {}) {
+  const createInterface =
+    deps.createInterface ||
+    (async () => {
+      const readline = await import('readline');
+      return readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+      });
+    });
+
+  const rl = await createInterface();
+  try {
+    return await new Promise((resolve) => {
+      rl.question(
+        'Would you like to continue with unsupported Python version? (y/n): ',
+        resolve
+      );
+    });
+  } finally {
+    rl.close();
+  }
+}

package/src/clients/cli/api_clients/REST/__init__.py

@@ -1,3 +0,0 @@
-from .sydney_client import SydneyClient
-
-__all__ = ["SydneyClient"]