@codeyam/codeyam-cli 0.1.0-staging.8aea589 → 0.1.0-staging.ad31e3e

package/analyzer-template/.build-info.json

@@ -1,10 +1,10 @@
 {
-  "buildTimestamp": "2025-12-09T21:27:34.619Z",
-  "buildTime": 1765315654619,
-  "gitCommit": "8aea589922b9c184fd5458f522d785af2e348e72",
+  "buildTimestamp": "2025-12-16T15:34:02.479Z",
+  "buildTime": 1765899242479,
+  "gitCommit": "ad31e3ef5ec259a22ee009ca6a7bc35682c297a7",
   "nodeVersion": "v20.19.6",
-  "contentHash": "131ce1b86de761d815eb9d47d321d6959a13627b88b736f0e01a873d6f68e719",
-  "buildNumber": 47,
-  "semanticVersion": "0.1.47",
-  "version": "0.1.47 (2025-12-09T21:27+131ce1b)"
+  "contentHash": "72c7943edf0ece740338c522a5b1737e7586cb2bee94c4bb8c3dfc320cc4f63f",
+  "buildNumber": 99,
+  "semanticVersion": "0.1.99",
+  "version": "0.1.99 (2025-12-16T15:34+72c7943)"
 }

package/analyzer-template/log.txt

@@ -1,7 +1,7 @@
 
-[12/9/2025, 9:27:34 PM] > codeyam-combo@1.0.0 mergeDependencies
-[12/9/2025, 9:27:34 PM] > node ./scripts/mergePackageJsonFiles.cjs
+[12/16/2025, 3:34:02 PM] > codeyam-combo@1.0.0 mergeDependencies
+[12/16/2025, 3:34:02 PM] > node ./scripts/mergePackageJsonFiles.cjs
 
 
-[12/9/2025, 9:27:34 PM] Merged dependencies into root package.json
+[12/16/2025, 3:34:02 PM] Merged dependencies into root package.json
 

package/analyzer-template/package.json

@@ -9,14 +9,14 @@
   "dependencies": {
     "@aws-sdk/client-cloudwatch-logs": "^3.940.0",
     "@aws-sdk/client-cloudfront": "^3.899.0",
-    "@aws-sdk/client-codebuild": "^3.940.0",
+    "@aws-sdk/client-codebuild": "^3.948.0",
     "@aws-sdk/client-dynamodb": "^3.899.0",
     "@aws-sdk/client-ec2": "^3.899.0",
-    "@aws-sdk/client-ecr": "^3.946.0",
+    "@aws-sdk/client-ecr": "^3.948.0",
     "@aws-sdk/client-ecs": "^3.937.0",
-    "@aws-sdk/client-s3": "^3.899.0",
+    "@aws-sdk/client-s3": "^3.940.0",
     "@aws-sdk/client-sqs": "^3.940.0",
-    "@aws-sdk/lib-storage": "^3.899.0",
+    "@aws-sdk/lib-storage": "^3.940.0",
     "@aws-sdk/util-dynamodb": "^3.902.0",
     "@octokit/auth-app": "^8.1.0",
     "@octokit/rest": "^22.0.0",
@@ -24,11 +24,11 @@
     "@supabase/supabase-js": "2.52.1",
     "domhandler": "^5.0.3",
     "dotenv": "^17.2.3",
-    "express": "^5.2.0",
+    "express": "^5.2.1",
     "get-port": "^7.1.0",
     "htmlparser2": "^10.0.0",
     "jest": "^30.2.0",
-    "jsdom": "^26.1.0",
+    "jsdom": "^27.3.0",
     "jsonc-parser": "^3.2.1",
     "lru-cache": "^11.2.4",
     "openai": "^6.9.0",
@@ -43,6 +43,7 @@
     "uuid": "^11.1.0",
     "pluralize": "^8.0.0",
     "@anthropic-ai/sdk": "^0.71.0",
+    "@aws-sdk/s3-request-presigner": "^3.940.0",
     "@octokit/request": "^10.0.3",
     "better-sqlite3": "^12.4.1",
     "fetch-retry": "^6.0.0",
@@ -54,7 +55,7 @@
   "devDependencies": {
     "typescript": "^5.9.3",
     "@jest/types": "^30.2.0",
-    "@types/jsdom": "^21.1.7",
+    "@types/jsdom": "^27.0.0",
     "@octokit/types": "^15.0.0",
     "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^30.0.0",

package/analyzer-template/packages/ai/index.ts

@@ -13,7 +13,6 @@ export { default as generateEntityDocumentation } from './src/lib/generateEntity
 export { default as generateChangesEntityDocumentation } from './src/lib/generateChangesEntityDocumentation';
 export { default as codeQualityEntityAnalysis } from './src/lib/codeQualityEntityAnalysis';
 export { default as summarizeDiff } from './src/lib/summarizeDiff';
-export { default as generateEntityDataMap } from './src/lib/generateEntityDataMap';
 export { default as generateEntityDataStructure } from './src/lib/generateEntityDataStructure';
 export { default as generateEntityScenarios } from './src/lib/generateEntityScenarios';
 export {
@@ -67,6 +66,7 @@ export type {
   SerializableFunctionCallInfo,
   SerializableFunctionResult,
   SerializableScopeVariable,
+  EnrichedConditionalUsage,
 } from './src/lib/worker/SerializableDataStructure';
 export {
   getExternalFunctionCallInfo,
@@ -75,6 +75,7 @@ export {
   getUsageEquivalencies,
   getSourceEquivalencies,
   getEquivalentSignatureVariables,
+  getConditionalUsages,
 } from './src/lib/worker/SerializableDataStructure';
 export { parseJsonSafe } from './src/lib/parsers/parseJsonSafe';
 export { default as completionCall } from './src/lib/completionCall';

package/analyzer-template/packages/ai/package.json

@@ -10,13 +10,13 @@
   "license": "ISC",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.71.0",
-    "jsdom": "^26.1.0",
+    "jsdom": "^27.3.0",
     "p-queue": "^8.1.0",
     "p-retry": "^6.2.1",
     "piscina": "^5.1.4",
     "lru-cache": "^11.2.4"
   },
   "devDependencies": {
-    "@types/jsdom": "^21.1.7"
+    "@types/jsdom": "^27.0.0"
   }
 }

package/analyzer-template/packages/ai/scripts/ai-test-matrix.mjs

@@ -0,0 +1,424 @@
+#!/usr/bin/env node
+/**
+ * AI Test Matrix Runner
+ *
+ * Runs AI test suites across multiple models with configurable concurrency.
+ * Outputs JSON results for analysis.
+ *
+ * Usage:
+ *   node packages/ai/scripts/ai-test-matrix.mjs [options]
+ *
+ * Options:
+ *   --test, -t       Test file pattern (default: all ai_tests)
+ *   --models, -m     Comma-separated model list (default: see MODELS below)
+ *   --runs, -r       Number of runs per model (default: 1, useful for flakiness detection)
+ *   --concurrency    Max concurrent test runs (default: 4)
+ *   --output, -o     Output file for results JSON (default: stdout)
+ *
+ * Examples:
+ *   node packages/ai/scripts/ai-test-matrix.mjs -t generateStatementAnalysis
+ *   node packages/ai/scripts/ai-test-matrix.mjs -m "gpt-4.1-mini,claude-sonnet-4-20250514" -r 3
+ */
+
+import { spawn } from 'node:child_process';
+import { writeFile } from 'node:fs/promises';
+import { parseArgs } from 'node:util';
+
+const MODELS = [
+  // OpenAI
+  'openai/gpt-5',
+  'openai/gpt-5-mini',
+  'openai/gpt-5-nano',
+  'openai/gpt-4.1',
+  'openai/gpt-4.1-mini',
+  'openai/gpt-4o',
+  'openai/gpt-4o-mini',
+
+  // Groq
+  'openai/gpt-oss-120b-groq',
+
+  // DeepInfra
+  'openai/gpt-oss-120b-deepinfra',
+  'qwen/qwen3-235b-instruct-deepinfra',
+  'qwen/qwen3-coder-480b-instruct-deepinfra',
+  'google/gemini-2.5-pro-deepinfra',
+  'google/gemini-2.5-flash-deepinfra',
+
+  // OpenRouter
+  'google/gemini-2.5-flash-lite-openrouter',
+  'meta-llama/llama-4-maverick-openrouter',
+  'deepseek/v3.1-terminus-openrouter',
+
+  // Anthropic
+  'anthropic/claude-4.5-haiku',
+  'anthropic/claude-4.5-sonnet',
+  'anthropic/claude-4.5-opus',
+];
+
+const AI_TESTS_DIR = 'packages/ai/src/lib/__tests__/ai_tests';
+
+// Tests excluded from default runs (low signal-to-noise ratio)
+const EXCLUDED_TESTS = [
+  'generateStatementAnalysis',
+  'generateEntityDataStructure',
+];
+
+const HELP_TEXT = `
+AI Test Matrix Runner
+
+Runs AI test suites across multiple models with configurable concurrency.
+Outputs JSON results for analysis.
+
+Usage:
+  node packages/ai/scripts/ai-test-matrix.mjs [options]
+
+Options:
+  --test, -t       Test file pattern (default: all ai_tests)
+  --models, -m     Comma-separated model list (default: ${MODELS.join(', ')})
+  --runs, -r       Number of runs per model (default: 1, useful for flakiness detection)
+  --concurrency    Max concurrent test runs (default: 4)
+  --output, -o     Output file for results JSON (default: stdout)
+  --help, -h       Show this help message
+
+Examples:
+  node packages/ai/scripts/ai-test-matrix.mjs -t generateStatementAnalysis
+  node packages/ai/scripts/ai-test-matrix.mjs -m "gpt-4.1-mini,claude-sonnet-4-20250514" -r 3
+`;
+
+function parseCliArgs() {
+  const { values } = parseArgs({
+    options: {
+      test: { type: 'string', short: 't', default: '' },
+      models: { type: 'string', short: 'm', default: '' },
+      runs: { type: 'string', short: 'r', default: '1' },
+      concurrency: { type: 'string', default: '4' },
+      output: { type: 'string', short: 'o', default: '' },
+      help: { type: 'boolean', short: 'h', default: false },
+    },
+  });
+
+  if (values.help) {
+    console.log(HELP_TEXT);
+    process.exit(0);
+  }
+
+  return {
+    testPattern: values.test || '',
+    models: values.models ? values.models.split(',').map((m) => m.trim()) : MODELS,
+    runs: parseInt(values.runs, 10),
+    concurrency: parseInt(values.concurrency, 10),
+    outputFile: values.output || '',
+  };
+}
+
+/**
+ * Run jest with JSON output and return parsed results
+ */
+async function runJest(model, testPattern, ignorePatterns = []) {
+  const testPath = testPattern
+    ? `${AI_TESTS_DIR}/${testPattern}.test.ts`
+    : AI_TESTS_DIR;
+
+  const jestArgs = ['jest', '--config', 'jest.config.ai.ts', '--json', testPath];
+
+  // Add --testPathIgnorePatterns for excluded tests
+  for (const pattern of ignorePatterns) {
+    jestArgs.push('--testPathIgnorePatterns', pattern);
+  }
+
+  return new Promise((resolve) => {
+    const startTime = Date.now();
+    const proc = spawn('npx', jestArgs, {
+        env: { ...process.env, CODEYAM_TEST_MODEL: model },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+
+    let stdout = '';
+    let stderr = '';
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+    proc.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    proc.on('close', (exitCode) => {
+      const duration = Date.now() - startTime;
+
+      // Jest outputs JSON to stdout, but may include non-JSON lines
+      // Find the JSON object in the output
+      let jsonResult = null;
+      try {
+        const jsonMatch = stdout.match(/^\{[\s\S]*\}$/m);
+        if (jsonMatch) {
+          jsonResult = JSON.parse(jsonMatch[0]);
+        }
+      } catch {
+        // JSON parse failed
+      }
+
+      if (jsonResult) {
+        resolve({
+          success: exitCode === 0,
+          model,
+          duration,
+          numTotalTests: jsonResult.numTotalTests,
+          numPassedTests: jsonResult.numPassedTests,
+          numFailedTests: jsonResult.numFailedTests,
+          testResults: jsonResult.testResults.map((tr) => {
+            const passed = tr.assertionResults.filter((ar) => ar.status === 'passed').length;
+            const failed = tr.assertionResults.filter((ar) => ar.status === 'failed').length;
+            return {
+              file: tr.name.replace(/.*ai_tests\//, ''),
+              passed,
+              failed,
+              failures: tr.assertionResults
+                .filter((ar) => ar.status === 'failed')
+                .map((ar) => ({
+                  name: ar.title,
+                  message: ar.failureMessages?.[0]?.slice(0, 500) || '',
+                })),
+            };
+          }),
+        });
+      } else {
+        resolve({
+          success: false,
+          model,
+          duration,
+          error: 'Failed to parse Jest JSON output',
+          stderr: stderr.slice(0, 1000),
+        });
+      }
+    });
+  });
+}
+
+/**
+ * Run tasks with limited concurrency
+ */
+async function runWithConcurrency(tasks, concurrency) {
+  const results = [];
+  const executing = new Set();
+
+  for (const task of tasks) {
+    const promise = task().then((result) => {
+      executing.delete(promise);
+      return result;
+    });
+    executing.add(promise);
+    results.push(promise);
+
+    if (executing.size >= concurrency) {
+      await Promise.race(executing);
+    }
+  }
+
+  return Promise.all(results);
+}
+
+/**
+ * Aggregate results for summary statistics
+ */
+function aggregateResults(results, models, runs) {
+  const byModel = {};
+
+  for (const model of models) {
+    const modelResults = results.filter((r) => r.model === model);
+    const cleanRuns = modelResults.filter((r) => r.numFailedTests === 0).length;
+    const totalPassed = modelResults.reduce((sum, r) => sum + (r.numPassedTests || 0), 0);
+    const totalTests = modelResults.reduce((sum, r) => sum + (r.numTotalTests || 0), 0);
+
+    byModel[model] = {
+      runs: modelResults.length,
+      cleanRuns,
+      testsTotal: totalTests,
+      testsPassed: totalPassed,
+      testPassRate: totalTests > 0 ? ((totalPassed / totalTests) * 100).toFixed(1) + '%' : 'N/A',
+      avgDuration: Math.round(
+        modelResults.reduce((sum, r) => sum + r.duration, 0) / modelResults.length
+      ),
+      // Flakiness: tests that passed in some runs but failed in others
+      flakyTests: runs > 1 ? findFlakyTests(modelResults) : [],
+    };
+  }
+
+  return byModel;
+}
+
+/**
+ * Find tests that passed in some runs but failed in others
+ */
+function findFlakyTests(modelResults) {
+  const testOutcomes = new Map();
+
+  for (const result of modelResults) {
+    if (!result.testResults) continue;
+
+    for (const tr of result.testResults) {
+      for (const failure of tr.failures || []) {
+        const key = `${tr.file}:${failure.name}`;
+        if (!testOutcomes.has(key)) {
+          testOutcomes.set(key, { passed: 0, failed: 0 });
+        }
+        testOutcomes.get(key).failed++;
+      }
+    }
+  }
+
+  // Mark passed for tests not in failures
+  const totalRuns = modelResults.length;
+  const flaky = [];
+
+  for (const [test, outcomes] of testOutcomes) {
+    outcomes.passed = totalRuns - outcomes.failed;
+    if (outcomes.passed > 0 && outcomes.failed > 0) {
+      flaky.push({
+        test,
+        passRate: ((outcomes.passed / totalRuns) * 100).toFixed(0) + '%',
+      });
+    }
+  }
+
+  return flaky;
+}
+
+async function main() {
+  const config = parseCliArgs();
+
+  console.error(`🧪 AI Test Matrix Runner`);
+  console.error(`   Models: ${config.models.join(', ')}`);
+  console.error(`   Runs per model: ${config.runs}`);
+  console.error(`   Concurrency: ${config.concurrency}`);
+  console.error(`   Test pattern: ${config.testPattern || '(all ai_tests)'}`);
+
+  // When running all tests (no specific pattern), exclude noisy tests
+  const ignorePatterns = config.testPattern ? [] : EXCLUDED_TESTS;
+  if (ignorePatterns.length > 0) {
+    console.error(`   Excluding: ${ignorePatterns.join(', ')}`);
+  }
+  console.error('');
+
+  // Build task list
+  const tasks = [];
+  for (const model of config.models) {
+    for (let run = 0; run < config.runs; run++) {
+      tasks.push(async () => {
+        const runLabel = config.runs > 1 ? ` (run ${run + 1}/${config.runs})` : '';
+        console.error(`⏳ Starting: ${model}${runLabel}`);
+        const result = await runJest(model, config.testPattern, ignorePatterns);
+        const status = result.numFailedTests === 0 ? '✅' : '❌';
+        console.error(
+          `${status} Finished: ${model}${runLabel} - ${result.numPassedTests}/${result.numTotalTests} passed (${result.duration}ms)`
+        );
+        return { ...result, runIndex: run };
+      });
+    }
+  }
+
+  const results = await runWithConcurrency(tasks, config.concurrency);
+  const summary = aggregateResults(results, config.models, config.runs);
+
+  const output = {
+    timestamp: new Date().toISOString(),
+    config: {
+      testPattern: config.testPattern || '(all)',
+      models: config.models,
+      runs: config.runs,
+    },
+    summary,
+    results,
+  };
+
+  const jsonOutput = JSON.stringify(output, null, 2);
+
+  if (config.outputFile) {
+    await writeFile(config.outputFile, jsonOutput);
+    console.error(`\n📄 Results written to ${config.outputFile}`);
+  } else {
+    console.log(jsonOutput);
+  }
+
+  // Print summary table to stderr
+  console.error('\n📊 Summary:');
+  console.error('─'.repeat(70));
+  for (const [model, stats] of Object.entries(summary)) {
+    console.error(
+      `${model.padEnd(40)} ${stats.testPassRate.padStart(6)} (${stats.testsPassed}/${stats.testsTotal}) | ${stats.avgDuration}ms`
+    );
+    if (stats.flakyTests.length > 0) {
+      console.error(`   ⚠️  Flaky: ${stats.flakyTests.map((f) => f.test).join(', ')}`);
+    }
+  }
+  console.error('─'.repeat(70));
+
+  // Print matrix view (models × test files)
+  printMatrix(results, config.models);
+}
+
+/**
+ * Print a matrix view: test files (rows) × models (columns)
+ */
+function printMatrix(results, models) {
+  // Collect all test files across all results
+  const testFiles = new Set();
+  for (const result of results) {
+    if (!result.testResults) continue;
+    for (const tr of result.testResults) {
+      testFiles.add(tr.file);
+    }
+  }
+
+  if (testFiles.size === 0) return;
+
+  const sortedFiles = [...testFiles].sort();
+
+  // Compute column widths
+  const fileColWidth = Math.max(30, ...sortedFiles.map((f) => f.replace('.test.ts', '').length + 2));
+  const modelColWidth = 12;
+
+  // Shorten model names for display
+  const shortModelName = (m) => {
+    const parts = m.split('/');
+    return parts[parts.length - 1].slice(0, modelColWidth - 1);
+  };
+
+  console.error('\n📋 Matrix (test files × models):');
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+
+  // Header row
+  const header = ''.padEnd(fileColWidth) + models.map((m) => shortModelName(m).padStart(modelColWidth)).join(' ');
+  console.error(header);
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+
+  // Data rows
+  for (const file of sortedFiles) {
+    const shortFile = file.replace('.test.ts', '');
+    let row = shortFile.padEnd(fileColWidth);
+
+    for (const model of models) {
+      // Find result for this model (use first run if multiple)
+      const result = results.find((r) => r.model === model);
+      const tr = result?.testResults?.find((t) => t.file === file);
+
+      if (tr) {
+        const total = tr.passed + tr.failed;
+        const cell = `${tr.passed}/${total}`;
+        row += cell.padStart(modelColWidth) + ' ';
+      } else {
+        row += '-'.padStart(modelColWidth) + ' ';
+      }
+    }
+
+    console.error(row);
+  }
+
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

package/analyzer-template/packages/ai/src/lib/analyzeScope.ts

@@ -160,6 +160,16 @@ export async function analyzeScopeLocal({
 
     const llmCalls: any[] = [];
     const allEnvironmentVariables: Set<string> = new Set();
+    // Collect conditional usages from all statement analyses
+    const allConditionalUsages: Record<
+      string,
+      Array<{
+        path: string;
+        conditionType: 'truthiness' | 'comparison' | 'switch';
+        comparedValues?: string[];
+        location: 'if' | 'ternary' | 'logical-and' | 'switch';
+      }>
+    > = {};
 
     const astAnalysisResultWithStatements = statementScopes.map(
       (statementScope) => {
@@ -171,6 +181,15 @@ export async function analyzeScopeLocal({
         astAnalysisResult.environmentVariables.forEach((envVar) =>
           allEnvironmentVariables.add(envVar),
         );
+        // Collect conditional usages from AST analysis
+        for (const [path, usages] of Object.entries(
+          astAnalysisResult.conditionalUsages,
+        )) {
+          if (!allConditionalUsages[path]) {
+            allConditionalUsages[path] = [];
+          }
+          allConditionalUsages[path].push(...usages);
+        }
         return {
           statementScope,
           astAnalysisResult,
@@ -432,6 +451,11 @@ export async function analyzeScopeLocal({
       );
     }
 
+    // Add conditional usages from this scope's analysis
+    if (Object.keys(allConditionalUsages).length > 0) {
+      scopeDataStructure.addConditionalUsages(allConditionalUsages);
+    }
+
     await Promise.all(
       Object.values(scope.childScopes).map(async (childScope) => {
         // This is awkward but need to identify class scopes

package/analyzer-template/packages/ai/src/lib/astScopes/astScopeAnalyzer.ts

@@ -170,17 +170,6 @@ export class ASTScopeAnalyzer {
     // console.info('SOURCEFILE', this.sourceFile.getText());
     // console.info('RANGE', this.statementInfo.start, this.statementInfo.end, this.sourceFile.getText().slice(this.statementInfo.start, this.statementInfo.end));
 
-    // console.info('RELEVANT NODES:\n\n', this.statementInfo.text);
-    // relevantNodes.forEach((node, i) =>
-    //   console.info(`RELEVANT NODE ${i}`, node.getText(this.sourceFile)),
-    // );
-
-    // console.info(
-    //   'CHILD BOUNDARIES',
-    //   this.statementInfo.childBoundaries.map((b) =>
-    //     this.sourceFile.getText().slice(b.start, b.end),
-    //   ),
-    // );
     this.processNodes(relevantNodes, context);
 
     // for (const childBoundary of this.statementInfo.childBoundaries) {
@@ -285,12 +274,13 @@ export class ASTScopeAnalyzer {
     node: ts.Node,
     sourceFile: ts.SourceFile,
   ): { nodeStart: number; nodeEnd: number } {
-    const sourceText = sourceFile.text;
-    const sourceGetText = sourceFile.getText();
-    const leadingContentOffset = sourceText.length - sourceGetText.length;
+    // Use getStart(sourceFile) to get the position after leading trivia.
+    // Note: We intentionally don't adjust for leadingContentOffset here because
+    // the positions from isolateStatements are calculated using getStart() too,
+    // which uses the same sourceFile.text coordinate system.
     return {
-      nodeStart: node.getStart(sourceFile) - leadingContentOffset,
-      nodeEnd: node.getEnd() - leadingContentOffset,
+      nodeStart: node.getStart(sourceFile),
+      nodeEnd: node.getEnd(),
     };
   }