agentic-qe 3.6.9 → 3.6.10

package/scripts/benchmark-kg-assisted.ts

@@ -0,0 +1,725 @@
+#!/usr/bin/env tsx
+/**
+ * KG-Assisted QE Benchmark (Issue #266)
+ *
+ * A/B comparison: does code intelligence KG improve QE agent test generation?
+ *
+ * Protocol:
+ *   Run A (Control)  — aqe init --minimal (no code intelligence) → test generate × N files
+ *   Run B (Treatment) — aqe init --auto   (KG populated)         → same test generate calls
+ *
+ * Metrics captured per file:
+ *   - Token usage (via `aqe token-usage --json`)
+ *   - Generated test count, assertions, coverage estimate
+ *   - Wall-clock latency
+ *   - Output size (chars — proxy for context consumed)
+ *   - KG index stats (vectors in memory.db)
+ *
+ * Test subject: https://github.com/maxritter/claude-pilot (cloned to /tmp)
+ *
+ * Run:
+ *   npx tsx scripts/benchmark-kg-assisted.ts
+ *   npx tsx scripts/benchmark-kg-assisted.ts --skip-clone   # reuse existing clone
+ *   npx tsx scripts/benchmark-kg-assisted.ts --files 3      # fewer files for quick test
+ */
+
+import { execSync, ExecSyncOptions } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+// ── Configuration ────────────────────────────────────────────────────
+
+const REPO_URL = 'https://github.com/maxritter/claude-pilot';
+const BENCH_DIR = '/tmp/kg-benchmark';
+const PROJECT_DIR = path.join(BENCH_DIR, 'claude-pilot');
+const RESULTS_DIR = path.join(BENCH_DIR, 'results');
+const AQE_DIR = path.join(PROJECT_DIR, '.agentic-qe');
+
+// Parse CLI args
+const args = process.argv.slice(2);
+const skipClone = args.includes('--skip-clone');
+const fileCountArg = args.indexOf('--files');
+const maxFiles = fileCountArg >= 0 ? parseInt(args[fileCountArg + 1] || '10', 10) : 10;
+
+// ── File Selection ───────────────────────────────────────────────────
+
+interface BenchmarkFile {
+  relativePath: string;
+  complexity: 'simple' | 'medium' | 'complex';
+  lineCount: number;
+}
+
+function discoverBenchmarkFiles(): BenchmarkFile[] {
+  const result = execSync(
+    `find ${PROJECT_DIR} -name '*.py' ! -path '*/test*' ! -name 'conftest.py' ! -name '__init__.py' -exec wc -l {} + 2>/dev/null | sort -rn | grep -v total`,
+    { encoding: 'utf-8' }
+  );
+
+  const files: BenchmarkFile[] = [];
+  for (const line of result.trim().split('\n')) {
+    const match = line.trim().match(/^(\d+)\s+(.+)$/);
+    if (!match) continue;
+    const lineCount = parseInt(match[1], 10);
+    const absPath = match[2];
+    const relativePath = path.relative(PROJECT_DIR, absPath);
+
+    let complexity: BenchmarkFile['complexity'];
+    if (lineCount < 150) complexity = 'simple';
+    else if (lineCount < 400) complexity = 'medium';
+    else complexity = 'complex';
+
+    files.push({ relativePath, complexity, lineCount });
+  }
+
+  // Select balanced set: 30% complex, 40% medium, 30% simple
+  const simple = files.filter((f) => f.complexity === 'simple');
+  const medium = files.filter((f) => f.complexity === 'medium');
+  const complex = files.filter((f) => f.complexity === 'complex');
+
+  const selected: BenchmarkFile[] = [];
+  const targets = [
+    { bucket: complex, target: Math.min(3, Math.ceil(maxFiles * 0.3)) },
+    { bucket: medium, target: Math.min(4, Math.ceil(maxFiles * 0.4)) },
+    { bucket: simple, target: Math.min(3, Math.ceil(maxFiles * 0.3)) },
+  ];
+
+  for (const { bucket, target } of targets) {
+    selected.push(...bucket.slice(0, target));
+  }
+
+  return selected.slice(0, maxFiles);
+}
+
+// ── AQE CLI Helpers ──────────────────────────────────────────────────
+
+const EXEC_OPTS: ExecSyncOptions = {
+  cwd: PROJECT_DIR,
+  encoding: 'utf-8' as BufferEncoding,
+  timeout: 300_000,
+  stdio: ['pipe', 'pipe', 'pipe'],
+};
+
+function runAqe(subcommand: string): string {
+  const cmd = `npx agentic-qe ${subcommand}`;
+  console.log(`    $ ${cmd}`);
+  try {
+    return execSync(cmd, EXEC_OPTS) as string;
+  } catch (e: unknown) {
+    const err = e as { stdout?: string; stderr?: string; message?: string };
+    const output = (err.stdout || '') + '\n' + (err.stderr || '');
+    console.error(`    [WARN] ${err.message?.split('\n')[0]}`);
+    return output;
+  }
+}
+
+function wipeAqeData(): void {
+  if (fs.existsSync(AQE_DIR)) {
+    fs.rmSync(AQE_DIR, { recursive: true, force: true });
+    console.log('    Wiped .agentic-qe/');
+  }
+}
+
+// ── Token Usage ──────────────────────────────────────────────────────
+
+interface TokenSnapshot {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  patternsReused: number;
+  tokensSaved: number;
+}
+
+function captureTokenUsage(): TokenSnapshot {
+  const empty: TokenSnapshot = { inputTokens: 0, outputTokens: 0, totalTokens: 0, patternsReused: 0, tokensSaved: 0 };
+
+  // Try reading token metrics directly from memory.db (faster than spawning CLI)
+  const memoryDb = path.join(AQE_DIR, 'memory.db');
+  if (!fs.existsSync(memoryDb)) return empty;
+
+  try {
+    // Query the token_metrics table directly via sqlite3
+    const result = execSync(
+      `sqlite3 '${memoryDb}' "SELECT COALESCE(SUM(json_extract(value, '$.usage.inputTokens')),0), COALESCE(SUM(json_extract(value, '$.usage.outputTokens')),0), COALESCE(SUM(json_extract(value, '$.usage.totalTokens')),0) FROM kv_store WHERE namespace='token-metrics';" 2>/dev/null`,
+      { encoding: 'utf-8', timeout: 5_000, cwd: PROJECT_DIR }
+    ).trim();
+
+    if (result) {
+      const parts = result.split('|');
+      return {
+        inputTokens: parseInt(parts[0], 10) || 0,
+        outputTokens: parseInt(parts[1], 10) || 0,
+        totalTokens: parseInt(parts[2], 10) || 0,
+        patternsReused: 0,
+        tokensSaved: 0,
+      };
+    }
+  } catch { /* table may not exist */ }
+
+  // Fallback: try the pattern reuse count
+  try {
+    const patternResult = execSync(
+      `sqlite3 '${memoryDb}' "SELECT COUNT(*) FROM qe_patterns WHERE access_count > 0;" 2>/dev/null`,
+      { encoding: 'utf-8', timeout: 5_000, cwd: PROJECT_DIR }
+    ).trim();
+    return { ...empty, patternsReused: parseInt(patternResult, 10) || 0 };
+  } catch { /* ignore */ }
+
+  return empty;
+}
+
+// ── Output Parsing ───────────────────────────────────────────────────
+
+interface ParsedOutput {
+  testsGenerated: number;
+  assertions: number;
+  coverageEstimate: number;
+  patternsUsed: number;
+  /** Number of mock declarations generated from KG dependencies */
+  kgMocksGenerated: number;
+  /** Number of KG dependency imports detected */
+  kgDependenciesFound: number;
+  /** Number of KG similarity references */
+  kgSimilarityRefs: number;
+  /** Estimated file reads avoided by using KG (= dependency count) */
+  fileReadsAvoided: number;
+  /** Estimated prompt tokens (chars / 4 approximation) */
+  estimatedPromptTokens: number;
+  /** Estimated output tokens (output chars / 4) */
+  estimatedOutputTokens: number;
+}
+
+function countAssertions(code: string): number {
+  // Count actual assertion patterns in generated code across frameworks
+  const patterns = [
+    /assert\s+/g,                         // Python assert
+    /self\.assert\w+/g,                   // unittest assertions
+    /pytest\.raises/g,                    // pytest raises
+    /expect\s*\(/g,                       // Jest/Vitest expect()
+    /\.to(?:Be|Equal|Have|Throw|Match|Contain)/g, // Jest matchers
+    /\.should\./g,                        // Chai should
+    /test_\w+/g,                          // Python test functions (each is an assertion unit)
+  ];
+  let count = 0;
+  for (const p of patterns) {
+    const matches = code.match(p);
+    if (matches) count += matches.length;
+  }
+  return count;
+}
+
+function parseTestGenerationOutput(output: string, sourceChars: number): ParsedOutput {
+  // "Generated N tests"
+  const genMatch = output.match(/Generated\s+(\d+)\s+test/i);
+  const testsGenerated = genMatch ? parseInt(genMatch[1], 10) : 0;
+
+  // "Assertions: N" from CLI output
+  const assertMatch = output.match(/Assertions:\s*(\d+)/i);
+  let assertions = assertMatch ? parseInt(assertMatch[1], 10) : 0;
+
+  // If CLI didn't report assertions, count them from the generated code
+  if (assertions === 0) {
+    assertions = countAssertions(output);
+  }
+
+  // "Coverage Estimate: N%"
+  const covMatch = output.match(/Coverage\s+Estimate:\s*([\d.]+)%/i);
+  const coverageEstimate = covMatch ? parseFloat(covMatch[1]) : 0;
+
+  // Count pattern-related lines
+  const patternMatches = output.match(/pattern|SONA|DecisionTransformer/gi);
+  const patternsUsed = patternMatches ? patternMatches.length : 0;
+
+  // KG-specific metrics
+  // Mock declarations from KG: "vi.mock('...')" or "jest.mock('...')" or "@patch('...')"
+  const mockMatches = output.match(/(?:vi|jest)\.mock\(['"]|@patch\(['"]|from unittest\.mock import/g);
+  const kgMocksGenerated = mockMatches ? mockMatches.length : 0;
+
+  // KG dependency imports: lines mentioning "Knowledge Graph" or "dependency analysis"
+  const kgDepMatches = output.match(/Knowledge Graph|dependency analysis|Auto-generated mocks from/gi);
+  const kgDependenciesFound = kgDepMatches ? kgDepMatches.length : 0;
+
+  // KG similarity references
+  const kgSimMatches = output.match(/Similar.*module|similarity:|similar code/gi);
+  const kgSimilarityRefs = kgSimMatches ? kgSimMatches.length : 0;
+
+  // File reads avoided: each mock/dependency from KG = 1 file read the agent didn't need
+  const importMatches = output.match(/(?:vi|jest)\.mock\(['"]([^'"]+)['"]/g);
+  const patchMatches = output.match(/@patch\(['"]([^'"]+)['"]/g);
+  const fileReadsAvoided = (importMatches?.length || 0) + (patchMatches?.length || 0);
+
+  // Token estimation: ~4 chars per token for code
+  // Prompt = source code + template (~500 chars) + KG context
+  const kgContextChars = kgDependenciesFound * 80 + kgSimilarityRefs * 200; // rough estimate
+  const estimatedPromptTokens = Math.round((sourceChars + 500 + kgContextChars) / 4);
+  const estimatedOutputTokens = Math.round(output.length / 4);
+
+  return {
+    testsGenerated, assertions, coverageEstimate, patternsUsed,
+    kgMocksGenerated, kgDependenciesFound, kgSimilarityRefs,
+    fileReadsAvoided, estimatedPromptTokens, estimatedOutputTokens,
+  };
+}
+
+// ── Test Generation ──────────────────────────────────────────────────
+
+interface GenerationResult {
+  file: string;
+  complexity: string;
+  lineCount: number;
+  sourceChars: number;
+  wallTimeMs: number;
+  parsed: ParsedOutput;
+  outputLength: number;
+  rawOutput: string;
+}
+
+function generateTestsForFile(file: BenchmarkFile): GenerationResult {
+  const absPath = path.join(PROJECT_DIR, file.relativePath);
+  const sourceCode = fs.readFileSync(absPath, 'utf-8');
+  const sourceChars = sourceCode.length;
+
+  const t0 = performance.now();
+
+  // CLI: aqe test generate <file> --type unit --framework pytest
+  const escapedPath = file.relativePath.replace(/'/g, "'\\''");
+  let output: string;
+  try {
+    output = execSync(
+      `npx agentic-qe test generate '${escapedPath}' --type unit --framework pytest 2>&1`,
+      { ...EXEC_OPTS, timeout: 120_000 }
+    ) as string;
+  } catch (e: unknown) {
+    const err = e as { stdout?: string; stderr?: string };
+    output = (err.stdout || '') + '\n' + (err.stderr || '');
+  }
+
+  const wallTimeMs = performance.now() - t0;
+  const parsed = parseTestGenerationOutput(output, sourceChars);
+
+  return {
+    file: file.relativePath,
+    complexity: file.complexity,
+    lineCount: file.lineCount,
+    sourceChars,
+    wallTimeMs,
+    parsed,
+    outputLength: output.length,
+    rawOutput: output,
+  };
+}
+
+// ── KG Stats ─────────────────────────────────────────────────────────
+
+interface KGStats {
+  vectorCount: number;
+  patternCount: number;
+  hasKG: boolean;
+  indexDuration?: number;
+  dbSizeBytes: number;
+}
+
+function getKGStats(): KGStats {
+  const memoryDb = path.join(AQE_DIR, 'memory.db');
+  const empty: KGStats = { vectorCount: 0, patternCount: 0, hasKG: false, dbSizeBytes: 0 };
+
+  if (!fs.existsSync(memoryDb)) return empty;
+
+  try {
+    const dbSize = fs.statSync(memoryDb).size;
+
+    // KG data lives in `vectors` table (code index) and `kv_store` (KG nodes/edges)
+    const vectorResult = execSync(
+      `sqlite3 '${memoryDb}' "SELECT COUNT(*) FROM vectors;" 2>/dev/null`,
+      { encoding: 'utf-8' }
+    ).trim();
+
+    const kvResult = execSync(
+      `sqlite3 '${memoryDb}' "SELECT COUNT(*) FROM kv_store;" 2>/dev/null`,
+      { encoding: 'utf-8' }
+    ).trim();
+
+    // Also check qe_patterns for learned patterns
+    const patternResult = execSync(
+      `sqlite3 '${memoryDb}' "SELECT COUNT(*) FROM qe_patterns;" 2>/dev/null`,
+      { encoding: 'utf-8' }
+    ).trim();
+
+    const vectorCount = parseInt(vectorResult, 10) || 0;
+    const kvCount = parseInt(kvResult, 10) || 0;
+    const patternCount = parseInt(patternResult, 10) || 0;
+
+    return { vectorCount, patternCount: patternCount + kvCount, hasKG: vectorCount > 0, dbSizeBytes: dbSize };
+  } catch {
+    return empty;
+  }
+}
+
+// ── Benchmark Run ────────────────────────────────────────────────────
+
+interface RunResult {
+  label: string;
+  kgStats: KGStats;
+  initOutput: string;
+  initDurationMs: number;
+  files: GenerationResult[];
+  totalWallTimeMs: number;
+  aggregateTokens: TokenSnapshot;
+}
+
+function runBenchmark(label: string, files: BenchmarkFile[], useAutoInit: boolean): RunResult {
+  console.log(`\n${'═'.repeat(60)}`);
+  console.log(`  ${label}`);
+  console.log(`${'═'.repeat(60)}`);
+
+  // Step 1: Wipe and reinitialize
+  wipeAqeData();
+
+  const initCmd = useAutoInit ? 'init --auto' : 'init --minimal --skip-patterns';
+  const initT0 = performance.now();
+  const initOutput = runAqe(initCmd);
+  let initDurationMs = performance.now() - initT0;
+  console.log(`    Init completed in ${(initDurationMs / 1000).toFixed(1)}s`);
+
+  // For treatment: explicitly run code indexing to populate KG
+  if (useAutoInit) {
+    console.log('    Running code intelligence indexing...');
+    const indexT0 = performance.now();
+    const indexOutput = runAqe('code index .');
+    const indexMs = performance.now() - indexT0;
+    initDurationMs += indexMs;
+    console.log(`    Code index completed in ${(indexMs / 1000).toFixed(1)}s`);
+
+    // Log index results
+    const filesMatch = indexOutput.match(/Files indexed:\s*(\d+)/);
+    const nodesMatch = indexOutput.match(/Nodes created:\s*(\d+)/);
+    const edgesMatch = indexOutput.match(/Edges created:\s*(\d+)/);
+    if (filesMatch) console.log(`    Indexed: ${filesMatch[1]} files, ${nodesMatch?.[1] || '?'} nodes, ${edgesMatch?.[1] || '?'} edges`);
+  }
+
+  // Step 2: KG stats
+  const kgStats = getKGStats();
+  if (useAutoInit) kgStats.indexDuration = initDurationMs;
+  console.log(`    KG vectors: ${kgStats.vectorCount} | patterns: ${kgStats.patternCount} | DB: ${(kgStats.dbSizeBytes / 1024).toFixed(0)}KB`);
+
+  // Step 3: Generate tests per file
+  const results: GenerationResult[] = [];
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    console.log(`\n    [${i + 1}/${files.length}] ${file.relativePath} (${file.complexity}, ${file.lineCount} lines)`);
+
+    const result = generateTestsForFile(file);
+    results.push(result);
+
+    console.log(`      Tests: ${result.parsed.testsGenerated} | Assertions: ${result.parsed.assertions} | Coverage: ${result.parsed.coverageEstimate}% | Time: ${(result.wallTimeMs / 1000).toFixed(1)}s`);
+    console.log(`      Output: ${result.outputLength} chars | Patterns: ${result.parsed.patternsUsed}`);
+    console.log(`      Tokens (est): prompt=${result.parsed.estimatedPromptTokens} output=${result.parsed.estimatedOutputTokens} | KG mocks: ${result.parsed.kgMocksGenerated} | File reads avoided: ${result.parsed.fileReadsAvoided}`);
+  }
+
+  // Capture token usage once at end of run (avoids per-file process overhead)
+  console.log('\n    Capturing token usage...');
+  const aggregateTokens = captureTokenUsage();
+  console.log(`    Tokens: in=${aggregateTokens.inputTokens} out=${aggregateTokens.outputTokens} total=${aggregateTokens.totalTokens} saved=${aggregateTokens.tokensSaved}`);
+
+  return {
+    label,
+    kgStats,
+    initOutput,
+    initDurationMs,
+    files: results,
+    totalWallTimeMs: results.reduce((s, r) => s + r.wallTimeMs, 0),
+    aggregateTokens,
+  };
+}
+
+// ── Comparison ───────────────────────────────────────────────────────
+
+interface FileMetrics {
+  tests: number; assertions: number; coverage: number;
+  wallMs: number; outputChars: number; patterns: number;
+  kgMocks: number; fileReadsAvoided: number;
+  estPromptTokens: number; estOutputTokens: number;
+}
+
+interface FileComparison {
+  file: string;
+  complexity: string;
+  lineCount: number;
+  control: FileMetrics;
+  treatment: FileMetrics;
+  delta: FileMetrics;
+}
+
+function buildMetrics(r: GenerationResult): FileMetrics {
+  return {
+    tests: r.parsed.testsGenerated,
+    assertions: r.parsed.assertions,
+    coverage: r.parsed.coverageEstimate,
+    wallMs: r.wallTimeMs,
+    outputChars: r.outputLength,
+    patterns: r.parsed.patternsUsed,
+    kgMocks: r.parsed.kgMocksGenerated,
+    fileReadsAvoided: r.parsed.fileReadsAvoided,
+    estPromptTokens: r.parsed.estimatedPromptTokens,
+    estOutputTokens: r.parsed.estimatedOutputTokens,
+  };
+}
+
+function deltaMetrics(a: FileMetrics, b: FileMetrics): FileMetrics {
+  const d = {} as FileMetrics;
+  for (const k of Object.keys(a) as (keyof FileMetrics)[]) {
+    (d as any)[k] = (b[k] as number) - (a[k] as number);
+  }
+  return d;
+}
+
+function compare(control: RunResult, treatment: RunResult): FileComparison[] {
+  return control.files.map((c, i) => {
+    const t = treatment.files[i];
+    const cm = buildMetrics(c);
+    const tm = buildMetrics(t);
+    return {
+      file: c.file,
+      complexity: c.complexity,
+      lineCount: c.lineCount,
+      control: cm,
+      treatment: tm,
+      delta: deltaMetrics(cm, tm),
+    };
+  });
+}
+
+// ── Summary Printing ─────────────────────────────────────────────────
+
+function printSummary(control: RunResult, treatment: RunResult, comparisons: FileComparison[]): void {
+  const W = 76;
+  const hr = '═'.repeat(W);
+  const line = (s: string) => console.log(`║  ${s.padEnd(W - 4)}  ║`);
+  const blank = () => line('');
+
+  console.log(`\n╔${hr}╗`);
+  console.log(`║${'KG-ASSISTED QE BENCHMARK RESULTS'.padStart((W + 32) / 2).padEnd(W)}║`);
+  console.log(`╠${hr}╣`);
+
+  blank();
+  line('INITIALIZATION');
+  line(`  Control  (no KG): ${(control.initDurationMs / 1000).toFixed(1).padStart(6)}s | Vectors: ${control.kgStats.vectorCount} | DB: ${(control.kgStats.dbSizeBytes / 1024).toFixed(0)}KB`);
+  line(`  Treatment (KG):   ${(treatment.initDurationMs / 1000).toFixed(1).padStart(6)}s | Vectors: ${treatment.kgStats.vectorCount} | DB: ${(treatment.kgStats.dbSizeBytes / 1024).toFixed(0)}KB`);
+
+  blank();
+  line('PER-FILE COMPARISON');
+  console.log(`╟${'─'.repeat(W)}╢`);
+  line('File                       Ctrl    KG    ΔTests  ΔAssert  ΔCov%   ΔTime');
+  console.log(`╟${'─'.repeat(W)}╢`);
+
+  for (const c of comparisons) {
+    const name = path.basename(c.file).padEnd(26).slice(0, 26);
+    const ctrl = String(c.control.tests).padStart(4);
+    const kg = String(c.treatment.tests).padStart(5);
+    const dt = ((c.delta.tests >= 0 ? '+' : '') + c.delta.tests).padStart(7);
+    const da = ((c.delta.assertions >= 0 ? '+' : '') + c.delta.assertions).padStart(8);
+    const dc = ((c.delta.coverage >= 0 ? '+' : '') + c.delta.coverage.toFixed(1)).padStart(6);
+    const dw = ((c.delta.wallMs >= 0 ? '+' : '') + (c.delta.wallMs / 1000).toFixed(1) + 's').padStart(7);
+    line(`${name} ${ctrl} ${kg} ${dt} ${da} ${dc} ${dw}`);
+  }
+
+  console.log(`╟${'─'.repeat(W)}╢`);
+
+  // Aggregates
+  const sumCtrl = { tests: 0, assertions: 0, coverage: 0 };
+  const sumKg = { tests: 0, assertions: 0, coverage: 0 };
+  for (const c of comparisons) {
+    sumCtrl.tests += c.control.tests;
+    sumCtrl.assertions += c.control.assertions;
+    sumCtrl.coverage += c.control.coverage;
+    sumKg.tests += c.treatment.tests;
+    sumKg.assertions += c.treatment.assertions;
+    sumKg.coverage += c.treatment.coverage;
+  }
+
+  const n = comparisons.length || 1;
+  const ctrlTok = control.aggregateTokens;
+  const kgTok = treatment.aggregateTokens;
+
+  // Compute estimated token totals from per-file data
+  const estCtrlPrompt = comparisons.reduce((s, c) => s + c.control.estPromptTokens, 0);
+  const estCtrlOutput = comparisons.reduce((s, c) => s + c.control.estOutputTokens, 0);
+  const estKgPrompt = comparisons.reduce((s, c) => s + c.treatment.estPromptTokens, 0);
+  const estKgOutput = comparisons.reduce((s, c) => s + c.treatment.estOutputTokens, 0);
+  const totalKgMocks = comparisons.reduce((s, c) => s + c.treatment.kgMocks, 0);
+  const totalFileReadsAvoided = comparisons.reduce((s, c) => s + c.treatment.fileReadsAvoided, 0);
+
+  blank();
+  line('AGGREGATES');
+  line(`  Tests generated:  Control=${sumCtrl.tests}  KG=${sumKg.tests}  Δ=${sumKg.tests - sumCtrl.tests}`);
+  line(`  Total assertions: Control=${sumCtrl.assertions}  KG=${sumKg.assertions}  Δ=${sumKg.assertions - sumCtrl.assertions}`);
+  line(`  Avg coverage:     Control=${(sumCtrl.coverage / n).toFixed(1)}%  KG=${(sumKg.coverage / n).toFixed(1)}%`);
+  line(`  Token usage (DB): Control=${ctrlTok.totalTokens}  KG=${kgTok.totalTokens}  Δ=${kgTok.totalTokens - ctrlTok.totalTokens}`);
+  line(`  Tokens saved:     Control=${ctrlTok.tokensSaved}  KG=${kgTok.tokensSaved}`);
+  line(`  Patterns reused:  Control=${ctrlTok.patternsReused}  KG=${kgTok.patternsReused}`);
+  line(`  Wall time:        Control=${(control.totalWallTimeMs / 1000).toFixed(1)}s  KG=${(treatment.totalWallTimeMs / 1000).toFixed(1)}s`);
+
+  blank();
+  line('TOKEN EFFICIENCY (estimated ~4 chars/token)');
+  line(`  Prompt tokens:    Control=${estCtrlPrompt}  KG=${estKgPrompt}  Δ=${estKgPrompt - estCtrlPrompt}`);
+  line(`  Output tokens:    Control=${estCtrlOutput}  KG=${estKgOutput}  Δ=${estKgOutput - estCtrlOutput}`);
+  line(`  Total estimated:  Control=${estCtrlPrompt + estCtrlOutput}  KG=${estKgPrompt + estKgOutput}  Δ=${(estKgPrompt + estKgOutput) - (estCtrlPrompt + estCtrlOutput)}`);
+  line(`  KG mocks generated: ${totalKgMocks} (each = 1 fewer file read for agent)`);
+  line(`  File reads avoided: ${totalFileReadsAvoided} (dependency info from KG instead of fs)`);
+
+  blank();
+  line('KEY FINDINGS (Issue #266 Questions)');
+  const tokenDelta = kgTok.totalTokens - ctrlTok.totalTokens;
+  const estTokenDelta = (estKgPrompt + estKgOutput) - (estCtrlPrompt + estCtrlOutput);
+  line(`  Q1: KG reduces tokens?     ${tokenDelta < 0 ? 'YES (DB: saved ' + Math.abs(tokenDelta) + ')' : tokenDelta === 0 ? 'NO CHANGE (DB)' : 'NO (DB: +' + tokenDelta + ')'} | Est: ${estTokenDelta < 0 ? 'saved ' + Math.abs(estTokenDelta) : '+' + estTokenDelta}`);
+  line(`  Q2: KG improves quality?   ${sumKg.tests > sumCtrl.tests ? 'YES (+' + (sumKg.tests - sumCtrl.tests) + ' tests, +' + (sumKg.assertions - sumCtrl.assertions) + ' assertions)' : 'NO CHANGE or WORSE'}`);
+  line(`  Q3: Latency justified?     ${treatment.totalWallTimeMs < control.totalWallTimeMs * 1.5 ? 'YES (overhead < 50%)' : 'NEEDS REVIEW (>' + ((treatment.totalWallTimeMs / control.totalWallTimeMs - 1) * 100).toFixed(0) + '% slower)'}`);
+  line(`  Q4: File reads avoided?    ${totalFileReadsAvoided > 0 ? 'YES (' + totalFileReadsAvoided + ' reads avoided via KG deps)' : 'NO (KG deps not yet used for mock generation)'}`);
+
+  // Q5: By complexity
+  for (const cx of ['simple', 'medium', 'complex'] as const) {
+    const subset = comparisons.filter((c) => c.complexity === cx);
+    if (subset.length === 0) continue;
+    const ct = subset.reduce((s, c) => s + c.control.tests, 0);
+    const kt = subset.reduce((s, c) => s + c.treatment.tests, 0);
+    const ca = subset.reduce((s, c) => s + c.control.assertions, 0);
+    const ka = subset.reduce((s, c) => s + c.treatment.assertions, 0);
+    const fr = subset.reduce((s, c) => s + c.treatment.fileReadsAvoided, 0);
+    line(`  Q5 [${cx.padEnd(7)}]: tests ${ct}→${kt}  assertions ${ca}→${ka}  file reads avoided: ${fr}`);
+  }
+
+  blank();
+  console.log(`╚${hr}╝`);
+}
+
+// ── Main ─────────────────────────────────────────────────────────────
+
+async function main() {
+  console.log('╔══════════════════════════════════════════════════════════════╗');
+  console.log('║     KG-Assisted QE Benchmark (Issue #266)                  ║');
+  console.log('║     Test subject: github.com/maxritter/claude-pilot        ║');
+  console.log('╚══════════════════════════════════════════════════════════════╝');
+  console.log(`  Timestamp:   ${new Date().toISOString()}`);
+  console.log(`  Max files:   ${maxFiles}`);
+  console.log(`  Skip clone:  ${skipClone}`);
+
+  // Step 1: Clone
+  if (!skipClone) {
+    console.log('\n── 1. Cloning test project ──────────────────────────');
+    if (fs.existsSync(PROJECT_DIR)) {
+      fs.rmSync(PROJECT_DIR, { recursive: true, force: true });
+    }
+    fs.mkdirSync(BENCH_DIR, { recursive: true });
+    execSync(`git clone ${REPO_URL} ${PROJECT_DIR}`, { stdio: 'inherit' });
+  } else {
+    console.log('\n── 1. Using existing clone ──────────────────────────');
+    if (!fs.existsSync(PROJECT_DIR)) {
+      console.error(`ERROR: ${PROJECT_DIR} does not exist. Run without --skip-clone.`);
+      process.exit(1);
+    }
+  }
+
+  // Step 2: Select files
+  console.log('\n── 2. Selecting benchmark files ─────────────────────');
+  const files = discoverBenchmarkFiles();
+  console.log(`  Selected ${files.length} files:`);
+  for (const f of files) {
+    console.log(`    [${f.complexity.padEnd(7)}] ${f.relativePath} (${f.lineCount} lines)`);
+  }
+  if (files.length === 0) {
+    console.error('ERROR: No source files found');
+    process.exit(1);
+  }
+
+  // Step 3: Run A — Control
+  console.log('\n── 3. Run A: Control (no KG) ────────────────────────');
+  const control = runBenchmark('CONTROL (no KG)', files, false);
+
+  // Step 4: Run B — Treatment
+  console.log('\n── 4. Run B: Treatment (with KG) ────────────────────');
+  const treatment = runBenchmark('TREATMENT (KG-assisted)', files, true);
+
+  // Step 5: Compare & report
+  console.log('\n── 5. Results ───────────────────────────────────────');
+  const comparisons = compare(control, treatment);
+  printSummary(control, treatment, comparisons);
+
+  // Step 6: Save JSON
+  fs.mkdirSync(RESULTS_DIR, { recursive: true });
+  const resultsPath = path.join(RESULTS_DIR, `kg-benchmark-${Date.now()}.json`);
+
+  const fullResults = {
+    timestamp: new Date().toISOString(),
+    testSubject: REPO_URL,
+    filesCount: files.length,
+    control: {
+      label: control.label,
+      kgStats: control.kgStats,
+      initDurationMs: control.initDurationMs,
+      totalWallTimeMs: control.totalWallTimeMs,
+      aggregateTokens: control.aggregateTokens,
+      files: control.files.map((f) => ({
+        file: f.file, complexity: f.complexity, lineCount: f.lineCount, sourceChars: f.sourceChars,
+        wallTimeMs: f.wallTimeMs, outputLength: f.outputLength, parsed: f.parsed,
+      })),
+    },
+    treatment: {
+      label: treatment.label,
+      kgStats: treatment.kgStats,
+      initDurationMs: treatment.initDurationMs,
+      totalWallTimeMs: treatment.totalWallTimeMs,
+      aggregateTokens: treatment.aggregateTokens,
+      files: treatment.files.map((f) => ({
+        file: f.file, complexity: f.complexity, lineCount: f.lineCount, sourceChars: f.sourceChars,
+        wallTimeMs: f.wallTimeMs, outputLength: f.outputLength, parsed: f.parsed,
+      })),
+    },
+    comparisons,
+    keyFindings: {
+      q1_kg_reduces_tokens: treatment.aggregateTokens.totalTokens < control.aggregateTokens.totalTokens,
+      q2_kg_improves_quality: comparisons.reduce((s, c) => s + c.delta.tests, 0) > 0,
+      q3_latency_justified: treatment.totalWallTimeMs < control.totalWallTimeMs * 1.5,
+      q4_file_reads_avoided: comparisons.reduce((s, c) => s + c.treatment.fileReadsAvoided, 0),
+      q5_complexity_breakdown: Object.fromEntries(
+        (['simple', 'medium', 'complex'] as const).map((cx) => {
+          const subset = comparisons.filter((c) => c.complexity === cx);
+          return [cx, {
+            files: subset.length,
+            controlTests: subset.reduce((s, c) => s + c.control.tests, 0),
+            kgTests: subset.reduce((s, c) => s + c.treatment.tests, 0),
+            controlAssertions: subset.reduce((s, c) => s + c.control.assertions, 0),
+            kgAssertions: subset.reduce((s, c) => s + c.treatment.assertions, 0),
+            fileReadsAvoided: subset.reduce((s, c) => s + c.treatment.fileReadsAvoided, 0),
+          }];
+        })
+      ),
+      tokenEfficiency: {
+        controlEstPromptTokens: comparisons.reduce((s, c) => s + c.control.estPromptTokens, 0),
+        controlEstOutputTokens: comparisons.reduce((s, c) => s + c.control.estOutputTokens, 0),
+        kgEstPromptTokens: comparisons.reduce((s, c) => s + c.treatment.estPromptTokens, 0),
+        kgEstOutputTokens: comparisons.reduce((s, c) => s + c.treatment.estOutputTokens, 0),
+        totalKgMocksGenerated: comparisons.reduce((s, c) => s + c.treatment.kgMocks, 0),
+        totalFileReadsAvoided: comparisons.reduce((s, c) => s + c.treatment.fileReadsAvoided, 0),
+      },
+    },
+  };
+
+  fs.writeFileSync(resultsPath, JSON.stringify(fullResults, null, 2));
+  console.log(`\n  Results JSON: ${resultsPath}`);
+
+  // Save raw outputs for manual diff
+  const rawDir = path.join(RESULTS_DIR, 'raw');
+  fs.mkdirSync(rawDir, { recursive: true });
+  for (const r of control.files) {
+    fs.writeFileSync(path.join(rawDir, `control-${path.basename(r.file, '.py')}.txt`), r.rawOutput);
+  }
+  for (const r of treatment.files) {
+    fs.writeFileSync(path.join(rawDir, `treatment-${path.basename(r.file, '.py')}.txt`), r.rawOutput);
+  }
+  console.log(`  Raw outputs:  ${rawDir}/`);
+}
+
+main().catch((err) => {
+  console.error('Benchmark failed:', err);
+  process.exit(1);
+});