@codeyam/codeyam-cli 0.1.0-bleeding-edge.8afd3ee → 0.1.0-staging.09652b8

package/analyzer-template/.build-info.json

@@ -1,10 +1,10 @@
 {
-  "buildTimestamp": "2025-12-09T21:06:56.685Z",
-  "buildTime": 1765314416685,
-  "gitCommit": "8afd3ee021d3d54ad4b194a574ae3bcd6427b53f",
+  "buildTimestamp": "2025-12-11T19:58:38.049Z",
+  "buildTime": 1765483118049,
+  "gitCommit": "09652b804d2c3f3bc36b68a17478631e582f31af",
   "nodeVersion": "v20.19.6",
-  "contentHash": "131ce1b86de761d815eb9d47d321d6959a13627b88b736f0e01a873d6f68e719",
-  "buildNumber": 47,
-  "semanticVersion": "0.1.47",
-  "version": "0.1.47 (2025-12-09T21:06+131ce1b)"
+  "contentHash": "6e8c56946bdcea6f1963d242e2b564537497d45ca94380cb79761d60d176e37e",
+  "buildNumber": 70,
+  "semanticVersion": "0.1.70",
+  "version": "0.1.70 (2025-12-11T19:58+6e8c569)"
 }

package/analyzer-template/log.txt

@@ -1,7 +1,7 @@
 
-[12/9/2025, 9:06:56 PM] > codeyam-combo@1.0.0 mergeDependencies
-[12/9/2025, 9:06:56 PM] > node ./scripts/mergePackageJsonFiles.cjs
+[12/11/2025, 7:58:37 PM] > codeyam-combo@1.0.0 mergeDependencies
+[12/11/2025, 7:58:37 PM] > node ./scripts/mergePackageJsonFiles.cjs
 
 
-[12/9/2025, 9:06:56 PM] Merged dependencies into root package.json
+[12/11/2025, 7:58:38 PM] Merged dependencies into root package.json
 

package/analyzer-template/package.json

@@ -24,7 +24,7 @@
     "@supabase/supabase-js": "2.52.1",
     "domhandler": "^5.0.3",
     "dotenv": "^17.2.3",
-    "express": "^5.2.0",
+    "express": "^5.2.1",
     "get-port": "^7.1.0",
     "htmlparser2": "^10.0.0",
     "jest": "^30.2.0",

package/analyzer-template/packages/ai/index.ts

@@ -13,7 +13,6 @@ export { default as generateEntityDocumentation } from './src/lib/generateEntity
 export { default as generateChangesEntityDocumentation } from './src/lib/generateChangesEntityDocumentation';
 export { default as codeQualityEntityAnalysis } from './src/lib/codeQualityEntityAnalysis';
 export { default as summarizeDiff } from './src/lib/summarizeDiff';
-export { default as generateEntityDataMap } from './src/lib/generateEntityDataMap';
 export { default as generateEntityDataStructure } from './src/lib/generateEntityDataStructure';
 export { default as generateEntityScenarios } from './src/lib/generateEntityScenarios';
 export {

package/analyzer-template/packages/ai/scripts/ai-test-matrix.mjs

@@ -0,0 +1,424 @@
+#!/usr/bin/env node
+/**
+ * AI Test Matrix Runner
+ *
+ * Runs AI test suites across multiple models with configurable concurrency.
+ * Outputs JSON results for analysis.
+ *
+ * Usage:
+ *   node packages/ai/scripts/ai-test-matrix.mjs [options]
+ *
+ * Options:
+ *   --test, -t       Test file pattern (default: all ai_tests)
+ *   --models, -m     Comma-separated model list (default: see MODELS below)
+ *   --runs, -r       Number of runs per model (default: 1, useful for flakiness detection)
+ *   --concurrency    Max concurrent test runs (default: 4)
+ *   --output, -o     Output file for results JSON (default: stdout)
+ *
+ * Examples:
+ *   node packages/ai/scripts/ai-test-matrix.mjs -t generateStatementAnalysis
+ *   node packages/ai/scripts/ai-test-matrix.mjs -m "gpt-4.1-mini,claude-sonnet-4-20250514" -r 3
+ */
+
+import { spawn } from 'node:child_process';
+import { writeFile } from 'node:fs/promises';
+import { parseArgs } from 'node:util';
+
+const MODELS = [
+  // OpenAI
+  'openai/gpt-5',
+  'openai/gpt-5-mini',
+  'openai/gpt-5-nano',
+  'openai/gpt-4.1',
+  'openai/gpt-4.1-mini',
+  'openai/gpt-4o',
+  'openai/gpt-4o-mini',
+
+  // Groq
+  'openai/gpt-oss-120b-groq',
+
+  // DeepInfra
+  'openai/gpt-oss-120b-deepinfra',
+  'qwen/qwen3-235b-instruct-deepinfra',
+  'qwen/qwen3-coder-480b-instruct-deepinfra',
+  'google/gemini-2.5-pro-deepinfra',
+  'google/gemini-2.5-flash-deepinfra',
+
+  // OpenRouter
+  'google/gemini-2.5-flash-lite-openrouter',
+  'meta-llama/llama-4-maverick-openrouter',
+  'deepseek/v3.1-terminus-openrouter',
+
+  // Anthropic
+  'anthropic/claude-4.5-haiku',
+  'anthropic/claude-4.5-sonnet',
+  'anthropic/claude-4.5-opus',
+];
+
+const AI_TESTS_DIR = 'packages/ai/src/lib/__tests__/ai_tests';
+
+// Tests excluded from default runs (low signal-to-noise ratio)
+const EXCLUDED_TESTS = [
+  'generateStatementAnalysis',
+  'generateEntityDataStructure',
+];
+
+const HELP_TEXT = `
+AI Test Matrix Runner
+
+Runs AI test suites across multiple models with configurable concurrency.
+Outputs JSON results for analysis.
+
+Usage:
+  node packages/ai/scripts/ai-test-matrix.mjs [options]
+
+Options:
+  --test, -t       Test file pattern (default: all ai_tests)
+  --models, -m     Comma-separated model list (default: ${MODELS.join(', ')})
+  --runs, -r       Number of runs per model (default: 1, useful for flakiness detection)
+  --concurrency    Max concurrent test runs (default: 4)
+  --output, -o     Output file for results JSON (default: stdout)
+  --help, -h       Show this help message
+
+Examples:
+  node packages/ai/scripts/ai-test-matrix.mjs -t generateStatementAnalysis
+  node packages/ai/scripts/ai-test-matrix.mjs -m "gpt-4.1-mini,claude-sonnet-4-20250514" -r 3
+`;
+
+function parseCliArgs() {
+  const { values } = parseArgs({
+    options: {
+      test: { type: 'string', short: 't', default: '' },
+      models: { type: 'string', short: 'm', default: '' },
+      runs: { type: 'string', short: 'r', default: '1' },
+      concurrency: { type: 'string', default: '4' },
+      output: { type: 'string', short: 'o', default: '' },
+      help: { type: 'boolean', short: 'h', default: false },
+    },
+  });
+
+  if (values.help) {
+    console.log(HELP_TEXT);
+    process.exit(0);
+  }
+
+  return {
+    testPattern: values.test || '',
+    models: values.models ? values.models.split(',').map((m) => m.trim()) : MODELS,
+    runs: parseInt(values.runs, 10),
+    concurrency: parseInt(values.concurrency, 10),
+    outputFile: values.output || '',
+  };
+}
+
+/**
+ * Run jest with JSON output and return parsed results
+ */
+async function runJest(model, testPattern, ignorePatterns = []) {
+  const testPath = testPattern
+    ? `${AI_TESTS_DIR}/${testPattern}.test.ts`
+    : AI_TESTS_DIR;
+
+  const jestArgs = ['jest', '--config', 'jest.config.ai.ts', '--json', testPath];
+
+  // Add --testPathIgnorePatterns for excluded tests
+  for (const pattern of ignorePatterns) {
+    jestArgs.push('--testPathIgnorePatterns', pattern);
+  }
+
+  return new Promise((resolve) => {
+    const startTime = Date.now();
+    const proc = spawn('npx', jestArgs, {
+        env: { ...process.env, CODEYAM_TEST_MODEL: model },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+
+    let stdout = '';
+    let stderr = '';
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+    proc.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    proc.on('close', (exitCode) => {
+      const duration = Date.now() - startTime;
+
+      // Jest outputs JSON to stdout, but may include non-JSON lines
+      // Find the JSON object in the output
+      let jsonResult = null;
+      try {
+        const jsonMatch = stdout.match(/^\{[\s\S]*\}$/m);
+        if (jsonMatch) {
+          jsonResult = JSON.parse(jsonMatch[0]);
+        }
+      } catch {
+        // JSON parse failed
+      }
+
+      if (jsonResult) {
+        resolve({
+          success: exitCode === 0,
+          model,
+          duration,
+          numTotalTests: jsonResult.numTotalTests,
+          numPassedTests: jsonResult.numPassedTests,
+          numFailedTests: jsonResult.numFailedTests,
+          testResults: jsonResult.testResults.map((tr) => {
+            const passed = tr.assertionResults.filter((ar) => ar.status === 'passed').length;
+            const failed = tr.assertionResults.filter((ar) => ar.status === 'failed').length;
+            return {
+              file: tr.name.replace(/.*ai_tests\//, ''),
+              passed,
+              failed,
+              failures: tr.assertionResults
+                .filter((ar) => ar.status === 'failed')
+                .map((ar) => ({
+                  name: ar.title,
+                  message: ar.failureMessages?.[0]?.slice(0, 500) || '',
+                })),
+            };
+          }),
+        });
+      } else {
+        resolve({
+          success: false,
+          model,
+          duration,
+          error: 'Failed to parse Jest JSON output',
+          stderr: stderr.slice(0, 1000),
+        });
+      }
+    });
+  });
+}
+
+/**
+ * Run tasks with limited concurrency
+ */
+async function runWithConcurrency(tasks, concurrency) {
+  const results = [];
+  const executing = new Set();
+
+  for (const task of tasks) {
+    const promise = task().then((result) => {
+      executing.delete(promise);
+      return result;
+    });
+    executing.add(promise);
+    results.push(promise);
+
+    if (executing.size >= concurrency) {
+      await Promise.race(executing);
+    }
+  }
+
+  return Promise.all(results);
+}
+
+/**
+ * Aggregate results for summary statistics
+ */
+function aggregateResults(results, models, runs) {
+  const byModel = {};
+
+  for (const model of models) {
+    const modelResults = results.filter((r) => r.model === model);
+    const cleanRuns = modelResults.filter((r) => r.numFailedTests === 0).length;
+    const totalPassed = modelResults.reduce((sum, r) => sum + (r.numPassedTests || 0), 0);
+    const totalTests = modelResults.reduce((sum, r) => sum + (r.numTotalTests || 0), 0);
+
+    byModel[model] = {
+      runs: modelResults.length,
+      cleanRuns,
+      testsTotal: totalTests,
+      testsPassed: totalPassed,
+      testPassRate: totalTests > 0 ? ((totalPassed / totalTests) * 100).toFixed(1) + '%' : 'N/A',
+      avgDuration: Math.round(
+        modelResults.reduce((sum, r) => sum + r.duration, 0) / modelResults.length
+      ),
+      // Flakiness: tests that passed in some runs but failed in others
+      flakyTests: runs > 1 ? findFlakyTests(modelResults) : [],
+    };
+  }
+
+  return byModel;
+}
+
+/**
+ * Find tests that passed in some runs but failed in others
+ */
+function findFlakyTests(modelResults) {
+  const testOutcomes = new Map();
+
+  for (const result of modelResults) {
+    if (!result.testResults) continue;
+
+    for (const tr of result.testResults) {
+      for (const failure of tr.failures || []) {
+        const key = `${tr.file}:${failure.name}`;
+        if (!testOutcomes.has(key)) {
+          testOutcomes.set(key, { passed: 0, failed: 0 });
+        }
+        testOutcomes.get(key).failed++;
+      }
+    }
+  }
+
+  // Mark passed for tests not in failures
+  const totalRuns = modelResults.length;
+  const flaky = [];
+
+  for (const [test, outcomes] of testOutcomes) {
+    outcomes.passed = totalRuns - outcomes.failed;
+    if (outcomes.passed > 0 && outcomes.failed > 0) {
+      flaky.push({
+        test,
+        passRate: ((outcomes.passed / totalRuns) * 100).toFixed(0) + '%',
+      });
+    }
+  }
+
+  return flaky;
+}
+
+async function main() {
+  const config = parseCliArgs();
+
+  console.error(`🧪 AI Test Matrix Runner`);
+  console.error(`   Models: ${config.models.join(', ')}`);
+  console.error(`   Runs per model: ${config.runs}`);
+  console.error(`   Concurrency: ${config.concurrency}`);
+  console.error(`   Test pattern: ${config.testPattern || '(all ai_tests)'}`);
+
+  // When running all tests (no specific pattern), exclude noisy tests
+  const ignorePatterns = config.testPattern ? [] : EXCLUDED_TESTS;
+  if (ignorePatterns.length > 0) {
+    console.error(`   Excluding: ${ignorePatterns.join(', ')}`);
+  }
+  console.error('');
+
+  // Build task list
+  const tasks = [];
+  for (const model of config.models) {
+    for (let run = 0; run < config.runs; run++) {
+      tasks.push(async () => {
+        const runLabel = config.runs > 1 ? ` (run ${run + 1}/${config.runs})` : '';
+        console.error(`⏳ Starting: ${model}${runLabel}`);
+        const result = await runJest(model, config.testPattern, ignorePatterns);
+        const status = result.numFailedTests === 0 ? '✅' : '❌';
+        console.error(
+          `${status} Finished: ${model}${runLabel} - ${result.numPassedTests}/${result.numTotalTests} passed (${result.duration}ms)`
+        );
+        return { ...result, runIndex: run };
+      });
+    }
+  }
+
+  const results = await runWithConcurrency(tasks, config.concurrency);
+  const summary = aggregateResults(results, config.models, config.runs);
+
+  const output = {
+    timestamp: new Date().toISOString(),
+    config: {
+      testPattern: config.testPattern || '(all)',
+      models: config.models,
+      runs: config.runs,
+    },
+    summary,
+    results,
+  };
+
+  const jsonOutput = JSON.stringify(output, null, 2);
+
+  if (config.outputFile) {
+    await writeFile(config.outputFile, jsonOutput);
+    console.error(`\n📄 Results written to ${config.outputFile}`);
+  } else {
+    console.log(jsonOutput);
+  }
+
+  // Print summary table to stderr
+  console.error('\n📊 Summary:');
+  console.error('─'.repeat(70));
+  for (const [model, stats] of Object.entries(summary)) {
+    console.error(
+      `${model.padEnd(40)} ${stats.testPassRate.padStart(6)} (${stats.testsPassed}/${stats.testsTotal}) | ${stats.avgDuration}ms`
+    );
+    if (stats.flakyTests.length > 0) {
+      console.error(`   ⚠️  Flaky: ${stats.flakyTests.map((f) => f.test).join(', ')}`);
+    }
+  }
+  console.error('─'.repeat(70));
+
+  // Print matrix view (models × test files)
+  printMatrix(results, config.models);
+}
+
+/**
+ * Print a matrix view: test files (rows) × models (columns)
+ */
+function printMatrix(results, models) {
+  // Collect all test files across all results
+  const testFiles = new Set();
+  for (const result of results) {
+    if (!result.testResults) continue;
+    for (const tr of result.testResults) {
+      testFiles.add(tr.file);
+    }
+  }
+
+  if (testFiles.size === 0) return;
+
+  const sortedFiles = [...testFiles].sort();
+
+  // Compute column widths
+  const fileColWidth = Math.max(30, ...sortedFiles.map((f) => f.replace('.test.ts', '').length + 2));
+  const modelColWidth = 12;
+
+  // Shorten model names for display
+  const shortModelName = (m) => {
+    const parts = m.split('/');
+    return parts[parts.length - 1].slice(0, modelColWidth - 1);
+  };
+
+  console.error('\n📋 Matrix (test files × models):');
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+
+  // Header row
+  const header = ''.padEnd(fileColWidth) + models.map((m) => shortModelName(m).padStart(modelColWidth)).join(' ');
+  console.error(header);
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+
+  // Data rows
+  for (const file of sortedFiles) {
+    const shortFile = file.replace('.test.ts', '');
+    let row = shortFile.padEnd(fileColWidth);
+
+    for (const model of models) {
+      // Find result for this model (use first run if multiple)
+      const result = results.find((r) => r.model === model);
+      const tr = result?.testResults?.find((t) => t.file === file);
+
+      if (tr) {
+        const total = tr.passed + tr.failed;
+        const cell = `${tr.passed}/${total}`;
+        row += cell.padStart(modelColWidth) + ' ';
+      } else {
+        row += '-'.padStart(modelColWidth) + ' ';
+      }
+    }
+
+    console.error(row);
+  }
+
+  console.error('─'.repeat(fileColWidth + models.length * (modelColWidth + 1)));
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

package/analyzer-template/packages/ai/src/lib/astScopes/astScopeAnalyzer.ts

@@ -170,17 +170,6 @@ export class ASTScopeAnalyzer {
     // console.info('SOURCEFILE', this.sourceFile.getText());
     // console.info('RANGE', this.statementInfo.start, this.statementInfo.end, this.sourceFile.getText().slice(this.statementInfo.start, this.statementInfo.end));
 
-    // console.info('RELEVANT NODES:\n\n', this.statementInfo.text);
-    // relevantNodes.forEach((node, i) =>
-    //   console.info(`RELEVANT NODE ${i}`, node.getText(this.sourceFile)),
-    // );
-
-    // console.info(
-    //   'CHILD BOUNDARIES',
-    //   this.statementInfo.childBoundaries.map((b) =>
-    //     this.sourceFile.getText().slice(b.start, b.end),
-    //   ),
-    // );
     this.processNodes(relevantNodes, context);
 
     // for (const childBoundary of this.statementInfo.childBoundaries) {
@@ -285,12 +274,13 @@ export class ASTScopeAnalyzer {
     node: ts.Node,
     sourceFile: ts.SourceFile,
   ): { nodeStart: number; nodeEnd: number } {
-    const sourceText = sourceFile.text;
-    const sourceGetText = sourceFile.getText();
-    const leadingContentOffset = sourceText.length - sourceGetText.length;
+    // Use getStart(sourceFile) to get the position after leading trivia.
+    // Note: We intentionally don't adjust for leadingContentOffset here because
+    // the positions from isolateStatements are calculated using getStart() too,
+    // which uses the same sourceFile.text coordinate system.
     return {
-      nodeStart: node.getStart(sourceFile) - leadingContentOffset,
-      nodeEnd: node.getEnd() - leadingContentOffset,
+      nodeStart: node.getStart(sourceFile),
+      nodeEnd: node.getEnd(),
     };
   }