@codeyam/codeyam-cli 0.1.0-staging.8aea589 → 0.1.0-staging.8b51541

package/analyzer-template/packages/ai/scripts/ai-test-matrix.mjs

@@ -0,0 +1,424 @@
+#!/usr/bin/env node
+/**
+ * AI Test Matrix Runner
+ *
+ * Runs AI test suites across multiple models with configurable concurrency.
+ * Outputs JSON results for analysis.
+ *
+ * Usage:
+ *   node packages/ai/scripts/ai-test-matrix.mjs [options]
+ *
+ * Options:
+ *   --test, -t       Test file pattern (default: all ai_tests)
+ *   --models, -m     Comma-separated model list (default: see MODELS below)
+ *   --runs, -r       Number of runs per model (default: 1, useful for flakiness detection)
+ *   --concurrency    Max concurrent test runs (default: 4)
+ *   --output, -o     Output file for results JSON (default: stdout)
+ *
+ * Examples:
+ *   node packages/ai/scripts/ai-test-matrix.mjs -t generateStatementAnalysis
+ *   node packages/ai/scripts/ai-test-matrix.mjs -m "gpt-4.1-mini,claude-sonnet-4-20250514" -r 3
+ */
+
+import { spawn } from 'node:child_process';
+import { writeFile } from 'node:fs/promises';
+import { parseArgs } from 'node:util';
+
+const MODELS = [
+  // OpenAI
+  'openai/gpt-5',
+  'openai/gpt-5-mini',
+  'openai/gpt-5-nano',
+  'openai/gpt-4.1',
+  'openai/gpt-4.1-mini',
+  'openai/gpt-4o',
+  'openai/gpt-4o-mini',
+
+  // Groq
+  'openai/gpt-oss-120b-groq',
+
+  // DeepInfra
+  'openai/gpt-oss-120b-deepinfra',
+  'qwen/qwen3-235b-instruct-deepinfra',
+  'qwen/qwen3-coder-480b-instruct-deepinfra',
+  'google/gemini-2.5-pro-deepinfra',
+  'google/gemini-2.5-flash-deepinfra',
+
+  // OpenRouter
+  'google/gemini-2.5-flash-lite-openrouter',
+  'meta-llama/llama-4-maverick-openrouter',
+  'deepseek/v3.1-terminus-openrouter',
+
+  // Anthropic
+  'anthropic/claude-4.5-haiku',
+  'anthropic/claude-4.5-sonnet',
+  'anthropic/claude-4.5-opus',
+];
+
+const AI_TESTS_DIR = 'packages/ai/src/lib/__tests__/ai_tests';
+
+// Tests excluded from default runs (low signal-to-noise ratio)
+const EXCLUDED_TESTS = [
+  'generateStatementAnalysis',
+  'generateEntityDataStructure',
+];
+
+const HELP_TEXT = `
+AI Test Matrix Runner
+
+Runs AI test suites across multiple models with configurable concurrency.
+Outputs JSON results for analysis.
+
+Usage:
+  node packages/ai/scripts/ai-test-matrix.mjs [options]
+
+Options:
+  --test, -t       Test file pattern (default: all ai_tests)
+  --models, -m     Comma-separated model list (default: ${MODELS.join(', ')})
+  --runs, -r       Number of runs per model (default: 1, useful for flakiness detection)
+  --concurrency    Max concurrent test runs (default: 4)
+  --output, -o     Output file for results JSON (default: stdout)
+  --help, -h       Show this help message
+
+Examples:
+  node packages/ai/scripts/ai-test-matrix.mjs -t generateStatementAnalysis
+  node packages/ai/scripts/ai-test-matrix.mjs -m "gpt-4.1-mini,claude-sonnet-4-20250514" -r 3
+`;
+
+function parseCliArgs() {
+  const { values } = parseArgs({
+    options: {
+      test: { type: 'string', short: 't', default: '' },
+      models: { type: 'string', short: 'm', default: '' },
+      runs: { type: 'string', short: 'r', default: '1' },
+      concurrency: { type: 'string', default: '4' },
+      output: { type: 'string', short: 'o', default: '' },
+      help: { type: 'boolean', short: 'h', default: false },
+    },
+  });
+
+  if (values.help) {
+    console.log(HELP_TEXT);
+    process.exit(0);
+  }
+
+  return {
+    testPattern: values.test || '',
+    models: values.models ? values.models.split(',').map((m) => m.trim()) : MODELS,
+    runs: parseInt(values.runs, 10),
+    concurrency: parseInt(values.concurrency, 10),
+    outputFile: values.output || '',
+  };
+}
+
+/**
+ * Run jest with JSON output and return parsed results
+ */
+async function runJest(model, testPattern, ignorePatterns = []) {
+  const testPath = testPattern
+    ? `${AI_TESTS_DIR}/${testPattern}.test.ts`
+    : AI_TESTS_DIR;
+
+  const jestArgs = ['jest', '--config', 'jest.config.ai.ts', '--json', testPath];
+
+  // Add --testPathIgnorePatterns for excluded tests
+  for (const pattern of ignorePatterns) {
+    jestArgs.push('--testPathIgnorePatterns', pattern);
+  }
+
+  return new Promise((resolve) => {
+    const startTime = Date.now();
+    const proc = spawn('npx', jestArgs, {
+        env: { ...process.env, CODEYAM_TEST_MODEL: model },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+
+    let stdout = '';
+    let stderr = '';
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+    proc.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    proc.on('close', (exitCode) => {
+      const duration = Date.now() - startTime;
+
+      // Jest outputs JSON to stdout, but may include non-JSON lines
+      // Find the JSON object in the output
+      let jsonResult = null;
+      try {
+        const jsonMatch = stdout.match(/^\{[\s\S]*\}$/m);
+        if (jsonMatch) {
+          jsonResult = JSON.parse(jsonMatch[0]);
+        }
+      } catch {
+        // JSON parse failed
+      }
+
+      if (jsonResult) {
+        resolve({
+          success: exitCode === 0,
+          model,
+          duration,
+          numTotalTests: jsonResult.numTotalTests,
+          numPassedTests: jsonResult.numPassedTests,
+          numFailedTests: jsonResult.numFailedTests,
+          testResults: jsonResult.testResults.map((tr) => {
+            const passed = tr.assertionResults.filter((ar) => ar.status === 'passed').length;
+            const failed = tr.assertionResults.filter((ar) => ar.status === 'failed').length;
+            return {
+              file: tr.name.replace(/.*ai_tests\//, ''),
+              passed,
+              failed,
+              failures: tr.assertionResults
+                .filter((ar) => ar.status === 'failed')
+                .map((ar) => ({
+                  name: ar.title,
+                  message: ar.failureMessages?.[0]?.slice(0, 500) || '',
+                })),
+            };
+          }),
+        });
+      } else {
+        resolve({
+          success: false,
+          model,
+          duration,
+          error: 'Failed to parse Jest JSON output',
+          stderr: stderr.slice(0, 1000),
+        });
+      }
+    });
+  });
+}
+
+/**
+ * Run tasks with limited concurrency
+ */
+async function runWithConcurrency(tasks, concurrency) {
+  const results = [];
+  const executing = new Set();
+
+  for (const task of tasks) {
+    const promise = task().then((result) => {
+      executing.delete(promise);
+      return result;
+    });
+    executing.add(promise);
+    results.push(promise);
+
+    if (executing.size >= concurrency) {
+      await Promise.race(executing);
+    }
+  }
+
+  return Promise.all(results);
+}
+
+/**
+ * Aggregate results for summary statistics
+ */
+function aggregateResults(results, models, runs) {
+  const byModel = {};
+
+  for (const model of models) {
+    const modelResults = results.filter((r) => r.model === model);
+    const cleanRuns = modelResults.filter((r) => r.numFailedTests === 0).length;
+    const totalPassed = modelResults.reduce((sum, r) => sum + (r.numPassedTests || 0), 0);
+    const totalTests = modelResults.reduce((sum, r) => sum + (r.numTotalTests || 0), 0);
+
+    byModel[model] = {
+      runs: modelResults.length,
+      cleanRuns,
+      testsTotal: totalTests,
+      testsPassed: totalPassed,
+      testPassRate: totalTests > 0 ? ((totalPassed / totalTests) * 100).toFixed(1) + '%' : 'N/A',
+      avgDuration: Math.round(
+        modelResults.reduce((sum, r) => sum + r.duration, 0) / modelResults.length
+      ),
+      // Flakiness: tests that passed in some runs but failed in others
+      flakyTests: runs > 1 ? findFlakyTests(modelResults) : [],
+    };
+  }
+
+  return byModel;
+}
+
+/**
+ * Find tests that passed in some runs but failed in others
+ */
+function findFlakyTests(modelResults) {
+  const testOutcomes = new Map();
+
+  for (const result of modelResults) {
+    if (!result.testResults) continue;
+
+    for (const tr of result.testResults) {
+      for (const failure of tr.failures || []) {
+        const key = `${tr.file}:${failure.name}`;
+        if (!testOutcomes.has(key)) {
+          testOutcomes.set(key, { passed: 0, failed: 0 });
+        }
+        testOutcomes.get(key).failed++;
+      }
+    }
+  }
+
+  // Mark passed for tests not in failures
+  const totalRuns = modelResults.length;
+  const flaky = [];
+
+  for (const [test, outcomes] of testOutcomes) {
+    outcomes.passed = totalRuns - outcomes.failed;
+    if (outcomes.passed > 0 && outcomes.failed > 0) {
+      flaky.push({
+        test,
+        passRate: ((outcomes.passed / totalRuns) * 100).toFixed(0) + '%',
+      });
+    }
+  }
+
+  return flaky;
+}
+
+async function main() {
+  const config = parseCliArgs();
+
+  console.error(`🧪 AI Test Matrix Runner`);
+  console.error(`   Models: ${config.models.join(', ')}`);
+  console.error(`   Runs per model: ${config.runs}`);
+  console.error(`   Concurrency: ${config.concurrency}`);
+  console.error(`   Test pattern: ${config.testPattern || '(all ai_tests)'}`);
+
+  // When running all tests (no specific pattern), exclude noisy tests
+  const ignorePatterns = config.testPattern ? [] : EXCLUDED_TESTS;
+  if (ignorePatterns.length > 0) {
+    console.error(`   Excluding: ${ignorePatterns.join(', ')}`);
+  }
+  console.error('');
+
+  // Build task list
+  const tasks = [];
+  for (const model of config.models) {
+    for (let run = 0; run < config.runs; run++) {
+      tasks.push(async () => {
+        const runLabel = config.runs > 1 ? ` (run ${run + 1}/${config.runs})` : '';
+        console.error(`⏳ Starting: ${model}${runLabel}`);
+        const result = await runJest(model, config.testPattern, ignorePatterns);
+        const status = result.numFailedTests === 0 ? '✅' : '❌';
+        console.error(
+          `${status} Finished: ${model}${runLabel} - ${result.numPassedTests}/${result.numTotalTests} passed (${result.duration}ms)`
+        );
+        return { ...result, runIndex: run };
+      });
+    }
+  }
+
+  const results = await runWithConcurrency(tasks, config.concurrency);
+  const summary = aggregateResults(results, config.models, config.runs);
+
+  const output = {
+    timestamp: new Date().toISOString(),
+    config: {
+      testPattern: config.testPattern || '(all)',
+      models: config.models,
+      runs: config.runs,
+    },
+    summary,
+    results,
+  };
+
+  const jsonOutput = JSON.stringify(output, null, 2);
+
+  if (config.outputFile) {
+    await writeFile(config.outputFile, jsonOutput);
+    console.error(`\n📄 Results written to ${config.outputFile}`);
+  } else {
+    console.log(jsonOutput);
+  }
+
+  // Print summary table to stderr
+  console.error('\n📊 Summary:');
+  console.error('─'.repeat(70));
+  for (const [model, stats] of Object.entries(summary)) {
+    console.error(
+      `${model.padEnd(40)} ${stats.testPassRate.padStart(6)} (${stats.testsPassed}/${stats.testsTotal}) | ${stats.avgDuration}ms`
+    );
+    if (stats.flakyTests.length > 0) {
+      console.error(`   ⚠️  Flaky: ${stats.flakyTests.map((f) => f.test).join(', ')}`);
+    }
+  }
+  console.error('─'.repeat(70));
+
+  // Print matrix view (models × test files)
+  printMatrix(results, config.models);
+}
+
+/**
+ * Print a matrix view: test files (rows) × models (columns)
+ */
+function printMatrix(results, models) {
+  // Collect all test files across all results
+  const testFiles = new Set();
+  for (const result of results) {
+    if (!result.testResults) continue;
+    for (const tr of result.testResults) {
+      testFiles.add(tr.file);
+    }
+  }
+
+  if (testFiles.size === 0) return;
+
+  const sortedFiles = [...testFiles].sort();
+
+  // Compute column widths
+  const fileColWidth = Math.max(30, ...sortedFiles.map((f) => f.replace('.test.ts', '').length + 2));
+  const modelColWidth = 12;
+
+  // Shorten model names for display
+  const shortModelName = (m) => {
+    const parts = m.split('/');
+    return parts[parts.length - 1].slice(0, modelColWidth - 1);
+  };
+
+  console.error('\n📋 Matrix (test files × models):');
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+
+  // Header row
+  const header = ''.padEnd(fileColWidth) + models.map((m) => shortModelName(m).padStart(modelColWidth)).join(' ');
+  console.error(header);
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+
+  // Data rows
+  for (const file of sortedFiles) {
+    const shortFile = file.replace('.test.ts', '');
+    let row = shortFile.padEnd(fileColWidth);
+
+    for (const model of models) {
+      // Find result for this model (use first run if multiple)
+      const result = results.find((r) => r.model === model);
+      const tr = result?.testResults?.find((t) => t.file === file);
+
+      if (tr) {
+        const total = tr.passed + tr.failed;
+        const cell = `${tr.passed}/${total}`;
+        row += cell.padStart(modelColWidth) + ' ';
+      } else {
+        row += '-'.padStart(modelColWidth) + ' ';
+      }
+    }
+
+    console.error(row);
+  }
+
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

package/analyzer-template/packages/ai/src/lib/__mocks__/completionCall.ts

@@ -1,3 +1,5 @@
+import * as fs from 'fs';
+import * as path from 'path';
 import OpenAI from 'openai';
 import * as lib from '../../lib';
 import { AI } from '../../lib/types';
@@ -23,6 +25,121 @@ interface CompletionCallProps {
   attempts?: number;
 }
 
+// Track which fixture index to use for each type (for LLM replay)
+const callIndexByType: Record<string, number> = {};
+
+/**
+ * Replays a captured LLM call from test fixtures instead of making a real API call.
+ */
+function replayLlmCall(
+  type: string,
+  fixturesDir: string,
+): {
+  finishReason: string;
+  completion: string;
+  stats: lib.types.LlmCallStats;
+} {
+  console.log(
+    `CodeYam Test: Replaying LLM call for type '${type}' from ${fixturesDir}`,
+  );
+
+  // Ensure the fixtures directory exists
+  if (!fs.existsSync(fixturesDir)) {
+    throw new Error(
+      `No LLM fixture files found - directory does not exist: ${fixturesDir}`,
+    );
+  }
+
+  // Load all JSON files from fixtures directory
+  const files = fs
+    .readdirSync(fixturesDir)
+    .filter((f) => f.endsWith('.json'));
+
+  if (files.length === 0) {
+    throw new Error(`No LLM fixture files found in ${fixturesDir}`);
+  }
+
+  // Track calls of each type to handle multiple calls with same type
+  const callsByType: Record<string, any[]> = {};
+
+  for (const file of files) {
+    try {
+      const content = fs.readFileSync(path.join(fixturesDir, file), 'utf-8');
+      const call = JSON.parse(content);
+
+      if (!callsByType[call.prompt_type]) {
+        callsByType[call.prompt_type] = [];
+      }
+      callsByType[call.prompt_type].push(call);
+    } catch (error) {
+      console.warn(`Failed to parse LLM fixture file ${file}:`, error);
+    }
+  }
+
+  const matchingCalls = callsByType[type];
+
+  if (!matchingCalls || matchingCalls.length === 0) {
+    const availableTypes = Object.keys(callsByType).join(', ');
+    console.warn(
+      `CodeYam Test: No captured LLM call found for type '${type}'. Available types: ${availableTypes}`,
+    );
+    // Return empty response - callers should handle gracefully
+    // This allows new LLM call types to be added without breaking existing fixtures
+    return {
+      finishReason: 'stop',
+      completion: '{}',
+      stats: {
+        model: 'fixture-fallback',
+        prompt_type: type,
+        system_message: '',
+        prompt_text: '',
+        response: '{}',
+        input_tokens: 0,
+        output_tokens: 0,
+        cost: 0,
+      },
+    };
+  }
+
+  // Track call count per type and cycle through multiple responses
+  const callKey = `${fixturesDir}::${type}`;
+  if (!callIndexByType[callKey]) {
+    callIndexByType[callKey] = 0;
+  }
+  const callIndex = callIndexByType[callKey];
+  callIndexByType[callKey] = (callIndex + 1) % matchingCalls.length;
+
+  const call = matchingCalls[callIndex];
+
+  console.log(
+    `CodeYam Test: Replaying LLM response for '${type}' [${callIndex + 1}/${matchingCalls.length}]`,
+  );
+
+  // Extract the actual completion content from the OpenAI response
+  let completion: string;
+  try {
+    const responseObj = JSON.parse(call.response);
+    completion = responseObj.choices?.[0]?.message?.content || call.response;
+  } catch {
+    completion = call.response;
+  }
+
+  return {
+    finishReason: 'stop',
+    completion,
+    stats: {
+      model: call.model ?? 'fixture',
+      prompt_type: type,
+      system_message: call.system_message ?? '',
+      prompt_text: call.prompt_text ?? '',
+      response: call.response ?? '',
+      input_tokens: call.input_tokens ?? 0,
+      output_tokens: call.output_tokens ?? 0,
+      cost: call.cost ?? 0,
+    },
+  };
+}
+
 export default async function completionCall({
   type,
   systemMessage,
@@ -34,6 +151,11 @@ export default async function completionCall({
   // model = AI.Model.DEEPSEEK_CHAT,
   attempts = 0,
 }: CompletionCallProps) {
+  // Support for E2E testing with replayed LLM calls
+  if (process.env.CODEYAM_LLM_FIXTURES_DIR) {
+    return replayLlmCall(type, process.env.CODEYAM_LLM_FIXTURES_DIR);
+  }
+
   const modelInfo = getModelInfo(model);
   const apiKey = process.env[modelInfo.provider.apiKeyEnvVar];