@graphpilot-oss/graphpilot 0.0.1 → 0.1.0

package/bench/results/baseline.md

@@ -1,28 +0,0 @@
-# GraphPilot Benchmark — 2026-05-20T06:13:00.314Z
-
-Corpus: `<graphpilot-repo>`
-graphpilot v0.0.1
-Node v23.11.0 on darwin
-
-## Aggregate
-
-- Tasks run: **10**
-- F1 (avg): graphpilot **0.89** vs grep **0.42**
-- Bytes processed (total): graphpilot **721B** vs grep **528.1KB** (99.9% reduction)
-- Winner counts: graphpilot **7** · grep **1** · tie **2**
-- Expected-winner accuracy: **9/10** (90%)
-
-## Per-task
-
-| # | Task | GP F1 | Grep F1 | GP bytes | Grep bytes | Winner | Expected |
-|---|---|---|---|---|---|---|---|
-| t01-callers-analyzeImpact | Find every function that calls analyzeImpact | 1.00 | 0.00 | 18B | 48.8KB | graphpilot | graphpilot ✓ |
-| t02-callers-extractSymbols | Find every direct caller of extractSymbols | 1.00 | 0.00 | 44B | 43.6KB | graphpilot | graphpilot ✓ |
-| t03-callers-validateRootPath | Find every direct caller of validateRootPath | 1.00 | 0.00 | 49B | 48.5KB | graphpilot | graphpilot ✓ |
-| t04-recall-substring-parse | Find every symbol whose name contains "parse" | 1.00 | 0.50 | 65B | 148.1KB | graphpilot | graphpilot ✓ |
-| t05-kind-filter-interfaces | Enumerate all TypeScript interfaces under src/ | 1.00 | 1.00 | 342B | 88.9KB | tie | graphpilot ✗ |
-| t06-impact-extractSymbols-depth2 | Compute blast radius of changing extractSymbols (depth 2) | 0.92 | 0.00 | 99B | 43.6KB | graphpilot | graphpilot ✓ |
-| t07-tests-affected-parseFile | Identify test files that exercise parseFile (directly) | 1.00 | 0.33 | 25B | 48.8KB | graphpilot | graphpilot ✓ |
-| t08-recall-substring-args | Find every MCP-tool input-args interface | 1.00 | 0.48 | 75B | 33.3KB | graphpilot | graphpilot ✓ |
-| t09-recall-miss | Look up a symbol that does not exist (negative test) | 1.00 | 1.00 | 2B | 6.9KB | tie | tie ✓ |
-| t10-string-literal-MAX_FILE_BYTES | Find every literal occurrence of the constant name "MAX_FILE_BYTES" | 0.00 | 0.86 | 2B | 17.5KB | grep | grep ✓ |

package/bench/run-agent-tier-automated.ts

@@ -1,234 +0,0 @@
-/**
- * Automated Tier-B Agent Benchmark Runner
- *
- * Instead of running Claude Code GUI sessions manually, this script:
- * 1. Programmatically calls the same gp_* tools that an agent would
- * 2. Measures structural correctness (blast radius, callers, etc.)
- * 3. Simulates agent reasoning by checking if key data was present
- * 4. Produces the per-task metrics table
- *
- * This is a proxy for real LLM agent behavior; it measures tool quality
- * rather than agent reasoning quality. But it's reproducible and fast.
- */
-
-import * as fs from 'node:fs';
-import { GraphIndex } from '../src/query.js';
-import { loadGraph } from '../src/storage.js';
-import { analyzeImpact } from '../src/impact.js';
-import { TASKS } from './tasks.js';
-import { getChangedFiles, readGitInfo } from '../src/git.js';
-import type { SymbolRecord, CallEdge } from '../src/symbols.js';
-
-interface TaskMetrics {
-  taskId: string;
-  description: string;
-  kind: string;
-  success: boolean; // did GP find all ground-truth results?
-  recall: number; // |found ∩ truth| / |truth|
-  precision: number; // |found ∩ truth| / |found|
-  f1: number;
-  hallucinations: number; // results not in ground truth
-  evidenceAnchorsPresent: boolean; // all results have file:line @ sha
-  tokenEstimate: number; // rough proxy: response size in chars / 4
-  notes: string;
-}
-
-function formatProvenance(s: SymbolRecord, sha: string | null): string {
-  const shaTag = sha ? ` @ ${sha.slice(0, 7)}` : '';
-  return `${s.file}:${s.line}${shaTag}`;
-}
-
-async function runTask(idx: GraphIndex, taskId: string, graph: any): Promise<TaskMetrics> {
-  const task = TASKS.find((t) => t.id === taskId);
-  if (!task) throw new Error(`Task ${taskId} not found`);
-
-  const shortSha = graph.indexedSha ? graph.indexedSha.slice(0, 7) : null;
-
-  let found: SymbolRecord[] = [];
-  let responseText = '';
-  let success = false;
-
-  try {
-    switch (task.kind) {
-      case 'callers': {
-        const target = idx.resolveSymbol(task.query);
-        if (target) {
-          const edges = idx.callers(target.id, { limit: 100 });
-          found = edges
-            .map((e) => idx.findById(e.fromId))
-            .filter((s) => s !== null) as SymbolRecord[];
-          responseText = found
-            .map((s) => `${s.name} @ ${formatProvenance(s, shortSha)}`)
-            .join('\n');
-        }
-        break;
-      }
-
-      case 'impact': {
-        const target = idx.resolveSymbol(task.query);
-        if (target) {
-          const impact = analyzeImpact(idx, task.query);
-          if (impact) {
-            found = impact.directCallers.map((c) => c.symbol);
-            found = found.concat(impact.transitiveCallers.map((c) => c.symbol));
-            responseText = [
-              `Direct: ${impact.directCallers.map((c) => c.symbol.name).join(', ')}`,
-              `Transitive: ${impact.transitiveCallers.map((c) => c.symbol.name).join(', ')}`,
-              impact.directCallers
-                .map((c) => `  ${c.symbol.name} @ ${formatProvenance(c.symbol, shortSha)}`)
-                .join('\n'),
-            ].join('\n');
-          }
-        }
-        break;
-      }
-
-      case 'impact-since': {
-        // Differential impact — simulated with empty changed files (clean repo)
-        const target = idx.resolveSymbol(task.query);
-        if (target) {
-          const impact = analyzeImpact(idx, task.query, { changedFiles: new Set() });
-          if (impact) {
-            found = [];
-            responseText = `(filtered to 0 files changed since HEAD~1)`;
-          }
-        }
-        break;
-      }
-
-      case 'recall':
-      case 'recall-substring': {
-        found = idx.findByName(task.query, { substring: task.kind === 'recall-substring' });
-        responseText = found
-          .map((s) => `${s.name} (${s.kind}) @ ${formatProvenance(s, shortSha)}`)
-          .join('\n');
-        break;
-      }
-
-      case 'kind-filter': {
-        found = idx.findByKind(task.query as any);
-        responseText = found.map((s) => `${s.name} @ ${formatProvenance(s, shortSha)}`).join('\n');
-        break;
-      }
-
-      case 'tests-affected': {
-        const target = idx.resolveSymbol(task.query);
-        if (target) {
-          const edges = idx.callers(target.id, { limit: 100 });
-          found = edges
-            .map((e) => idx.findById(e.fromId))
-            .filter((s) => s !== null && s.file.includes('test')) as SymbolRecord[];
-          responseText = found.map((s) => `${s.file}:${s.line}`).join('\n');
-        }
-        break;
-      }
-
-      case 'recall-miss': {
-        found = idx.findByName(task.query);
-        responseText =
-          found.length === 0 ? '[not found in index]' : found.map((s) => s.name).join(', ');
-        break;
-      }
-
-      case 'string-literal': {
-        // We can't search text; skip this (would be grep-only)
-        found = [];
-        responseText = '[string search not implemented in GraphPilot]';
-        break;
-      }
-    }
-  } catch (err) {
-    const msg = err instanceof Error ? err.message : String(err);
-    responseText = `[ERROR: ${msg}]`;
-  }
-
-  // Score against ground truth
-  const truth = new Set(task.groundTruth);
-  const foundNames = new Set(found.map((s) => s.name));
-
-  const intersection = new Set([...foundNames].filter((n) => truth.has(n)));
-  const recall = truth.size > 0 ? intersection.size / truth.size : 1;
-  const precision = foundNames.size > 0 ? intersection.size / foundNames.size : 1;
-  const f1 = precision + recall > 0 ? (2 * (precision * recall)) / (precision + recall) : 0;
-
-  success = recall === 1 && precision === 1;
-  const hallucinations = foundNames.size - intersection.size;
-
-  // Check for evidence anchors in response
-  const evidenceAnchorsPresent = /:\d+\s*@\s*[0-9a-f]{7}/.test(responseText) || found.length === 0;
-
-  const tokenEstimate = Math.ceil(responseText.length / 4);
-
-  return {
-    taskId,
-    description: task.description,
-    kind: task.kind,
-    success,
-    recall: Math.round(recall * 100) / 100,
-    precision: Math.round(precision * 100) / 100,
-    f1: Math.round(f1 * 100) / 100,
-    hallucinations,
-    evidenceAnchorsPresent,
-    tokenEstimate,
-    notes: `truth=${Array.from(truth).join(',')} found=${Array.from(foundNames).join(',')}`,
-  };
-}
-
-async function main() {
-  // Load from the repo root (indexer stores it with the repo-relative path hash)
-  const repoPath = process.cwd();
-  const graph = loadGraph(repoPath);
-  if (!graph) {
-    console.error(`No graph.json found for ${repoPath}. Run: node dist/cli.js index .`);
-    process.exit(1);
-  }
-
-  const idx = new GraphIndex(graph);
-  const results: TaskMetrics[] = [];
-
-  console.log(`Running ${TASKS.length} tasks against indexed GraphPilot...\n`);
-
-  for (const task of TASKS) {
-    const metrics = await runTask(idx, task.id, graph);
-    results.push(metrics);
-    const icon = metrics.success ? '✓' : '✗';
-    console.log(
-      `${icon} ${metrics.taskId}: F1=${metrics.f1} recall=${metrics.recall} prec=${metrics.precision}`,
-    );
-  }
-
-  // Write results
-  const timestamp = new Date().toISOString().split('T')[0];
-  const resultsPath = `bench/results/agent-tier-${timestamp}.md`;
-
-  let md = `# Tier-B Benchmark Results (Automated)\n\n`;
-  md += `Timestamp: ${new Date().toISOString()}\n\n`;
-  md += `## Per-Task Metrics\n\n`;
-  md += `| Task | Description | Success | Recall | Precision | F1 | Halluc | Anchors |\n`;
-  md += `|---|---|---|---|---|---|---|---|\n`;
-
-  let totalSuccess = 0;
-  let totalHalluc = 0;
-
-  for (const m of results) {
-    const success = m.success ? '✓' : '✗';
-    const anchors = m.evidenceAnchorsPresent ? '✓' : '✗';
-    md += `| ${m.taskId} | ${m.description} | ${success} | ${m.recall} | ${m.precision} | ${m.f1} | ${m.hallucinations} | ${anchors} |\n`;
-    totalSuccess += m.success ? 1 : 0;
-    totalHalluc += m.hallucinations;
-  }
-
-  md += `\n## Summary\n\n`;
-  md += `- **Tasks passed:** ${totalSuccess}/${results.length}\n`;
-  md += `- **Total hallucinations:** ${totalHalluc}\n`;
-  md += `- **Evidence anchors:** ${results.filter((r) => r.evidenceAnchorsPresent).length}/${results.filter((r) => r.kind !== 'string-literal').length} (excluding string-search)\n`;
-  md += `- **Mean F1 across tasks:** ${(results.reduce((n, r) => n + r.f1, 0) / results.length).toFixed(2)}\n`;
-
-  fs.mkdirSync('bench/results', { recursive: true });
-  fs.writeFileSync(resultsPath, md);
-
-  console.log(`\nResults written to ${resultsPath}`);
-  console.log(`\n${md}`);
-}
-
-main().catch(console.error);

package/bench/run-agent-tier.md

@@ -1,125 +0,0 @@
-# Tier-B Agent Benchmark — Spec
-
-> The launch headline ("Claude Code with GraphPilot succeeded on X/10
-> refactor tasks vs Y/10 without") lives here. This is the **manual
-> turn-the-crank session** that produces those numbers.
->
-> **Status:** spec only. Numbers not yet produced. Tier A (in
-> [README.md](README.md)) covers the deterministic, tool-only
-> comparison. Tier B adds an LLM in the loop.
-
-## Why Tier B is separate
-
-Tier A measures _whether the tools return the right info_. Tier B
-measures _whether the agent reaches the right conclusion using those
-tools_. Both matter; they answer different questions.
-
-Tier A is automatable. Tier B is not — it requires:
-
-1. Running real Claude Code sessions
-2. Scoring "did the agent reach the right answer?" by hand
-3. Recording token usage from the agent's logs
-
-That's ~4–6 hours of focused human work. Out of scope for a single
-benchmark commit; in scope for a separate launch-prep session.
-
-## Method
-
-### Setup
-
-- A test repo (preferably `microsoft/TypeScript` — large enough to
-  matter, recognizable to readers)
-- Three Claude Code configurations:
-  - **Baseline:** vanilla Claude Code, no MCP servers
-  - **With GraphPilot:** Claude Code with the graphpilot MCP server
-    configured + a CLAUDE.md routing rule pointing structural questions
-    at the gp\_\* tools
-  - **With CodeGraphContext** (optional but punchy): the closest OSS
-    competitor, same setup
-
-### The 10 tasks
-
-These mirror the Tier-A corpus but are phrased as natural-language
-refactor prompts:
-
-1. Rename `createSourceFile` everywhere it's called
-2. Find every function that catches but ignores errors
-3. List the public API exported from `src/compiler/` (or pick one module)
-4. Find the shortest call path from `parser.ts` to a syscall (`fs.write*`)
-5. Find functions never called by any test
-6. Which functions take `Diagnostic` as a parameter?
-7. Find all callers of a function flagged `@deprecated`
-8. Locate the function that emits a specific error message text
-9. Trace a value from CLI input to where it's logged (expect agents to
-   fail this — taint analysis isn't our beat)
-10. Find HTTP routes without auth middleware (expect failure — no
-    framework-aware tooling in v1)
-
-Tasks 9 and 10 are **deliberate "graphpilot loses" tasks**. Including
-them is what keeps the result believable.
-
-### Metrics per task
-
-For each `(task, condition)` cell:
-
-| Metric                  | How                                                |
-| ----------------------- | -------------------------------------------------- |
-| **Task success** (0/1)  | Human eval against a hand-written rubric           |
-| **Hallucination count** | Manual count of fabricated names / paths / imports |
-| **Token cost**          | Sum of input+output tokens from Claude Code's log  |
-| **Wall-clock**          | Stopwatch from prompt-submit to final answer       |
-| **Clean patch apply**   | Did the proposed diff apply without conflict?      |
-
-### Scoring
-
-Aggregate the per-task numbers into the headline:
-
-```
-Claude Code alone:        N/10 tasks succeeded
-Claude Code + GraphPilot: M/10 tasks succeeded
-Token cost:               −X%
-Hallucinations:           −Y%
-```
-
-If reality comes back at 5/10 vs 4/10, publish that — don't fake it.
-
-## Runbook (when the session happens)
-
-1. Clone the corpus repo (e.g. `microsoft/TypeScript`) to a clean dir
-2. Configure Claude Code three ways (vanilla / + graphpilot / + CGC)
-3. For each task: open a fresh session in each config, paste the prompt,
-   run until Claude produces an answer or gives up, score the result
-4. Tally totals; write the per-task table into
-   `bench/results/agent-tier-<date>.md`
-5. Drop the headline into the project README
-
-## Why we haven't done this yet
-
-- Tier A produces real, publishable numbers in <1 minute and locks in
-  the methodology. Better to have that floor than to launch with no
-  numbers because Tier B is half-done.
-- Running Tier B costs real money (~$10–20 per pass in Claude tokens)
-  and ~4–6 hours of attention. Worth doing right, in a focused session,
-  not interleaved with development.
-- The Tier-A bytes-reduction number (99.9 %) is _already_ sufficient
-  for a Show HN headline: _"99% fewer tokens needed to answer
-  structural questions in your TypeScript codebase."_
-
-## Estimated effort
-
-- Setup: 30 min
-- Run + score: 3–4 hours (10 tasks × 3 conditions × ~6 min)
-- Writeup + numbers into README: 30 min
-
-Total: half a working day.
-
-## What to do if Tier-B numbers are mediocre
-
-If "Claude Code + GraphPilot" comes back at 6/10 vs 5/10 baseline, the
-honest move is:
-
-1. Publish the real number
-2. Reframe the launch around Tier A (where the win is huge)
-3. Investigate WHY the agent didn't translate tool quality into answer
-   quality (probably: tool descriptions not aggressive enough, or
-   CLAUDE.md routing not strong enough). Fix and re-run before launch.

package/bench/run-baseline-tier.ts

@@ -1,200 +0,0 @@
-/**
- * Baseline Tier-B: vanilla grep + CLI tools
- *
- * Simulates what an agent would do without GraphPilot:
- * - Use `grep -r` for queries
- * - No structured index, no blast-radius analysis
- * - High noise (false positives in comments, strings)
- *
- * This is a strawman baseline; real agents might use LSP or IDEs.
- * But grep represents the cost of *no* structured indexing.
- */
-
-import { execSync } from 'node:child_process';
-import * as fs from 'node:fs';
-import { TASKS } from './tasks.js';
-
-interface TaskMetrics {
-  taskId: string;
-  description: string;
-  kind: string;
-  success: boolean;
-  recall: number;
-  precision: number;
-  f1: number;
-  hallucinations: number;
-  tokenEstimate: number;
-  notes: string;
-}
-
-function runGrep(pattern: string, options: string[] = []): string[] {
-  try {
-    const cmd = [
-      'grep',
-      '-r',
-      '--include=*.ts',
-      '--include=*.tsx',
-      ...options,
-      pattern,
-      'src',
-      'tests',
-    ].join(' ');
-    const output = execSync(cmd, {
-      encoding: 'utf8',
-      cwd: '.',
-      stdio: ['pipe', 'pipe', 'ignore'],
-    }).trim();
-    if (!output) return [];
-    return output.split('\n').filter((l) => l);
-  } catch {
-    return [];
-  }
-}
-
-function extractSymbolsFromGrep(lines: string[]): Set<string> {
-  const results = new Set<string>();
-  for (const line of lines) {
-    const m = line.match(/\b([a-zA-Z_][a-zA-Z0-9_]*)\b/g);
-    if (m) m.forEach((n) => results.add(n));
-  }
-  return results;
-}
-
-async function runTask(taskId: string): Promise<TaskMetrics> {
-  const task = TASKS.find((t) => t.id === taskId);
-  if (!task) throw new Error(`Task ${taskId} not found`);
-
-  let found = new Set<string>();
-  let responseText = '';
-
-  try {
-    switch (task.kind) {
-      case 'callers': {
-        // grep for function call pattern (naive heuristic)
-        const lines = runGrep(`\\b${task.query}\\s*\\(`);
-        found = extractSymbolsFromGrep(lines);
-        responseText = lines.join('\n').slice(0, 500);
-        break;
-      }
-
-      case 'impact':
-      case 'impact-since': {
-        // Can't compute blast radius with grep — too many false positives
-        // Simulate by grepping for the function name everywhere
-        const lines = runGrep(`\\b${task.query}\\b`);
-        found = extractSymbolsFromGrep(lines);
-        responseText = `(grep can't compute blast radius; found ${lines.length} occurrences)`;
-        break;
-      }
-
-      case 'recall':
-      case 'recall-substring': {
-        const pattern = task.kind === 'recall-substring' ? task.query : `\\b${task.query}\\b`;
-        const lines = runGrep(pattern);
-        found = extractSymbolsFromGrep(lines);
-        responseText = lines.join('\n').slice(0, 500);
-        break;
-      }
-
-      case 'kind-filter': {
-        // grep for 'interface Foo', 'function bar', etc.
-        const lines = runGrep(`${task.query}\\s+[a-zA-Z_]`);
-        found = extractSymbolsFromGrep(lines);
-        responseText = lines.join('\n').slice(0, 500);
-        break;
-      }
-
-      case 'tests-affected': {
-        const lines = runGrep(`\\b${task.query}\\b`, ['tests']);
-        found = extractSymbolsFromGrep(lines);
-        responseText = lines.join('\n').slice(0, 500);
-        break;
-      }
-
-      case 'recall-miss': {
-        const lines = runGrep(`\\b${task.query}\\b`);
-        found = extractSymbolsFromGrep(lines);
-        responseText = found.size === 0 ? '[not found in grep]' : found.size.toString();
-        break;
-      }
-
-      case 'string-literal': {
-        const lines = runGrep(task.query);
-        found = new Set(lines.map((l) => l.split(':')[0])); // file paths
-        responseText = lines.join('\n').slice(0, 500);
-        break;
-      }
-    }
-  } catch (err) {
-    responseText = `[ERROR: ${err}]`;
-  }
-
-  // Score
-  const truth = new Set(task.groundTruth);
-  const intersection = new Set([...found].filter((n) => truth.has(n)));
-  const recall = truth.size > 0 ? intersection.size / truth.size : 1;
-  const precision = found.size > 0 ? intersection.size / found.size : 1;
-  const f1 = precision + recall > 0 ? (2 * (precision * recall)) / (precision + recall) : 0;
-
-  const success = recall === 1 && precision === 1;
-  const hallucinations = found.size - intersection.size;
-  const tokenEstimate = Math.ceil(responseText.length / 4);
-
-  return {
-    taskId,
-    description: task.description,
-    kind: task.kind,
-    success,
-    recall: Math.round(recall * 100) / 100,
-    precision: Math.round(precision * 100) / 100,
-    f1: Math.round(f1 * 100) / 100,
-    hallucinations,
-    tokenEstimate,
-    notes: `truth=${Array.from(truth).join(',')} found=${Array.from(found).join(',')}`,
-  };
-}
-
-async function main() {
-  const results: TaskMetrics[] = [];
-
-  console.log(`Running ${TASKS.length} tasks with grep baseline...\n`);
-
-  for (const task of TASKS) {
-    const metrics = await runTask(task.id);
-    results.push(metrics);
-    const icon = metrics.success ? '✓' : '✗';
-    console.log(
-      `${icon} ${metrics.taskId}: F1=${metrics.f1} recall=${metrics.recall} prec=${metrics.precision}`,
-    );
-  }
-
-  const timestamp = new Date().toISOString().split('T')[0];
-  const resultsPath = `bench/results/baseline-tier-${timestamp}.md`;
-
-  let md = `# Baseline Tier-B (grep)\n\n`;
-  md += `| Task | Description | Success | Recall | Precision | F1 | Halluc |\n`;
-  md += `|---|---|---|---|---|---|---|\n`;
-
-  let totalSuccess = 0;
-  let totalHalluc = 0;
-
-  for (const m of results) {
-    const success = m.success ? '✓' : '✗';
-    md += `| ${m.taskId} | ${m.description} | ${success} | ${m.recall} | ${m.precision} | ${m.f1} | ${m.hallucinations} |\n`;
-    totalSuccess += m.success ? 1 : 0;
-    totalHalluc += m.hallucinations;
-  }
-
-  md += `\n## Summary\n\n`;
-  md += `- **Tasks passed:** ${totalSuccess}/${results.length}\n`;
-  md += `- **Total hallucinations:** ${totalHalluc}\n`;
-  md += `- **Mean F1:** ${(results.reduce((n, r) => n + r.f1, 0) / results.length).toFixed(2)}\n`;
-
-  fs.mkdirSync('bench/results', { recursive: true });
-  fs.writeFileSync(resultsPath, md);
-
-  console.log(`\nResults written to ${resultsPath}`);
-  console.log(`\n${md}`);
-}
-
-main().catch(console.error);