@graphpilot-oss/graphpilot 0.0.1 → 0.1.0

package/bench/run.ts

@@ -1,210 +0,0 @@
-/**
- * Tier-A benchmark runner. Runs each task in TASKS in two conditions
- * (graphpilot, baseline grep), scores both, writes a JSON result file
- * plus a markdown summary to bench/results/.
- *
- * Usage:
- *   pnpm bench [--repo=<path>] [--out=<file>]
- *
- * Defaults to running against the graphpilot repo itself
- * (process.cwd()), which is the self-test corpus.
- */
-
-import { mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
-import { join, resolve } from 'node:path';
-import { TASKS, type Task } from './tasks.js';
-import { GraphpilotRunner, type RunResult } from './runner-graphpilot.js';
-import { BaselineRunner } from './runner-baseline.js';
-import { score, type Scored } from './score.js';
-
-interface PerTaskResult {
-  task: Task;
-  graphpilot: { run: RunResult; score: Scored };
-  baseline: { run: RunResult; score: Scored };
-  winner: 'graphpilot' | 'grep' | 'tie';
-  /** Did the winner match expectedWinner? Diagnostic. */
-  expectedMatch: boolean;
-}
-
-interface AggregateMetrics {
-  totalTasks: number;
-  graphpilotF1Sum: number;
-  baselineF1Sum: number;
-  graphpilotBytesTotal: number;
-  baselineBytesTotal: number;
-  graphpilotWins: number;
-  baselineWins: number;
-  ties: number;
-  expectedWinnerHits: number;
-}
-
-interface BenchmarkReport {
-  meta: {
-    corpus: string;
-    timestamp: string;
-    graphpilotVersion: string;
-    nodeVersion: string;
-    platform: NodeJS.Platform;
-  };
-  aggregate: AggregateMetrics;
-  perTask: PerTaskResult[];
-}
-
-function pickWinner(gp: Scored, bl: Scored): 'graphpilot' | 'grep' | 'tie' {
-  const epsilon = 0.001;
-  if (Math.abs(gp.f1 - bl.f1) < epsilon) return 'tie';
-  return gp.f1 > bl.f1 ? 'graphpilot' : 'grep';
-}
-
-function fmt(n: number, dp = 2): string {
-  return Number.isFinite(n) ? n.toFixed(dp) : '?';
-}
-
-function fmtBytes(n: number): string {
-  if (n < 1024) return `${n}B`;
-  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)}KB`;
-  return `${(n / 1024 / 1024).toFixed(1)}MB`;
-}
-
-function summaryMarkdown(report: BenchmarkReport): string {
-  const a = report.aggregate;
-  const lines: string[] = [];
-  lines.push(`# GraphPilot Benchmark — ${report.meta.timestamp}`);
-  lines.push('');
-  lines.push(`Corpus: \`${report.meta.corpus}\``);
-  lines.push(`graphpilot v${report.meta.graphpilotVersion}`);
-  lines.push(`Node ${report.meta.nodeVersion} on ${report.meta.platform}`);
-  lines.push('');
-  lines.push('## Aggregate');
-  lines.push('');
-  lines.push(`- Tasks run: **${a.totalTasks}**`);
-  lines.push(
-    `- F1 (avg): graphpilot **${fmt(a.graphpilotF1Sum / a.totalTasks)}** ` +
-      `vs grep **${fmt(a.baselineF1Sum / a.totalTasks)}**`,
-  );
-  lines.push(
-    `- Bytes processed (total): graphpilot **${fmtBytes(a.graphpilotBytesTotal)}** ` +
-      `vs grep **${fmtBytes(a.baselineBytesTotal)}**` +
-      ` (${fmt((1 - a.graphpilotBytesTotal / a.baselineBytesTotal) * 100, 1)}% reduction)`,
-  );
-  lines.push(
-    `- Winner counts: graphpilot **${a.graphpilotWins}** · grep **${a.baselineWins}** · tie **${a.ties}**`,
-  );
-  lines.push(
-    `- Expected-winner accuracy: **${a.expectedWinnerHits}/${a.totalTasks}** ` +
-      `(${fmt((a.expectedWinnerHits / a.totalTasks) * 100, 0)}%)`,
-  );
-  lines.push('');
-  lines.push('## Per-task');
-  lines.push('');
-  lines.push('| # | Task | GP F1 | Grep F1 | GP bytes | Grep bytes | Winner | Expected |');
-  lines.push('|---|---|---|---|---|---|---|---|');
-  for (const t of report.perTask) {
-    const match = t.expectedMatch ? '✓' : '✗';
-    lines.push(
-      `| ${t.task.id} | ${t.task.description} ` +
-        `| ${fmt(t.graphpilot.score.f1)} ` +
-        `| ${fmt(t.baseline.score.f1)} ` +
-        `| ${fmtBytes(t.graphpilot.run.outputBytes)} ` +
-        `| ${fmtBytes(t.baseline.run.outputBytes)} ` +
-        `| ${t.winner} ` +
-        `| ${t.task.expectedWinner} ${match} |`,
-    );
-  }
-  return lines.join('\n');
-}
-
-async function main(): Promise<number> {
-  const args = process.argv.slice(2);
-  let repoArg: string | undefined;
-  let outArg: string | undefined;
-  for (const a of args) {
-    if (a.startsWith('--repo=')) repoArg = a.slice('--repo='.length);
-    else if (a.startsWith('--out=')) outArg = a.slice('--out='.length);
-  }
-
-  const repo = resolve(repoArg ?? process.cwd());
-  const gp = new GraphpilotRunner(repo);
-  const baseline = new BaselineRunner(repo);
-
-  const perTask: PerTaskResult[] = [];
-  const agg: AggregateMetrics = {
-    totalTasks: TASKS.length,
-    graphpilotF1Sum: 0,
-    baselineF1Sum: 0,
-    graphpilotBytesTotal: 0,
-    baselineBytesTotal: 0,
-    graphpilotWins: 0,
-    baselineWins: 0,
-    ties: 0,
-    expectedWinnerHits: 0,
-  };
-
-  for (const task of TASKS) {
-    const gpRun = gp.run(task);
-    const blRun = baseline.run(task);
-
-    const gpScore = score(gpRun.returned, task.groundTruth);
-    const blScore = score(blRun.returned, task.groundTruth);
-    const winner = pickWinner(gpScore, blScore);
-    const expectedMatch = winner === task.expectedWinner;
-
-    perTask.push({
-      task,
-      graphpilot: { run: gpRun, score: gpScore },
-      baseline: { run: blRun, score: blScore },
-      winner,
-      expectedMatch,
-    });
-
-    agg.graphpilotF1Sum += gpScore.f1;
-    agg.baselineF1Sum += blScore.f1;
-    agg.graphpilotBytesTotal += gpRun.outputBytes;
-    agg.baselineBytesTotal += blRun.outputBytes;
-    if (winner === 'graphpilot') agg.graphpilotWins++;
-    else if (winner === 'grep') agg.baselineWins++;
-    else agg.ties++;
-    if (expectedMatch) agg.expectedWinnerHits++;
-  }
-
-  // Pull package.json version for the meta block.
-  const pkgVersion = JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8')).version as string;
-
-  const report: BenchmarkReport = {
-    meta: {
-      corpus: repo,
-      timestamp: new Date().toISOString(),
-      graphpilotVersion: pkgVersion,
-      nodeVersion: process.version,
-      platform: process.platform,
-    },
-    aggregate: agg,
-    perTask,
-  };
-
-  const resultsDir = join(repo, 'bench', 'results');
-  if (!existsSync(resultsDir)) mkdirSync(resultsDir, { recursive: true });
-
-  const ts = report.meta.timestamp.replace(/[:.]/g, '-');
-  const jsonPath = outArg ?? join(resultsDir, `bench-${ts}.json`);
-  const mdPath = jsonPath.replace(/\.json$/, '.md');
-
-  writeFileSync(jsonPath, JSON.stringify(report, null, 2), 'utf8');
-  writeFileSync(mdPath, summaryMarkdown(report), 'utf8');
-
-  // Console summary
-  console.log(summaryMarkdown(report));
-  console.log('');
-  console.log(`Wrote ${jsonPath}`);
-  console.log(`Wrote ${mdPath}`);
-
-  return 0;
-}
-
-main().then(
-  (code) => process.exit(code),
-  (err) => {
-    console.error('Benchmark failed:', err);
-    process.exit(1);
-  },
-);

package/bench/runner-baseline.ts

@@ -1,177 +0,0 @@
-/**
- * Grep-style baseline. Simulates what an agent without structural memory
- * would do: scan every source file for the query as a literal substring,
- * return the matching files / function names.
- *
- * This UNDERSTATES the real baseline cost because:
- *   - A real agent reads context around each grep hit (we count just
- *     the matching files' raw bytes)
- *   - A real agent often grep+read multiple times before answering
- *
- * Even with that bias toward grep, GraphPilot should still win the
- * structural tasks by a large margin on `outputBytes` (proxy for tokens
- * the agent would have to read).
- */
-
-import fg from 'fast-glob';
-import { readFileSync, statSync } from 'node:fs';
-import { relative, resolve } from 'node:path';
-import type { Task } from './tasks.js';
-
-export interface RunResult {
-  returned: string[];
-  /** Total bytes the agent would have to read to answer this question via grep. */
-  outputBytes: number;
-  durationMs: number;
-}
-
-const SOURCE_GLOB = ['**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx'];
-const IGNORE = [
-  '**/node_modules/**',
-  '**/dist/**',
-  '**/build/**',
-  '**/coverage/**',
-  '**/.next/**',
-  '**/.nuxt/**',
-  '**/*.d.ts',
-];
-
-/** Cache: filePath -> bytes + lines. Avoids re-reading on every task. */
-interface FileCache {
-  path: string;
-  rel: string;
-  bytes: number;
-  lines: string[];
-}
-
-export class BaselineRunner {
-  readonly absRoot: string;
-  private readonly files: FileCache[];
-
-  constructor(repoRoot: string) {
-    this.absRoot = resolve(repoRoot);
-    const filePaths = fg.sync(SOURCE_GLOB, {
-      cwd: this.absRoot,
-      absolute: true,
-      ignore: IGNORE,
-      onlyFiles: true,
-    });
-    this.files = filePaths.map((p) => {
-      const text = readFileSync(p, 'utf8');
-      return {
-        path: p,
-        rel: relative(this.absRoot, p),
-        bytes: statSync(p).size,
-        lines: text.split('\n'),
-      };
-    });
-  }
-
-  /**
-   * Scan all source files for `query` as a literal substring. Returns
-   * (a) the set of matching files (for string-literal/tests-affected
-   * tasks) and (b) function-like identifier names that appear adjacent
-   * to `function`/`class`/`interface`/`const` keywords (a rough proxy
-   * for "what an agent would conclude").
-   */
-  private grepScan(query: string): {
-    matchedFiles: Set<string>;
-    bytesRead: number;
-    suspectedNames: Set<string>;
-  } {
-    const matchedFiles = new Set<string>();
-    const suspectedNames = new Set<string>();
-    let bytesRead = 0;
-
-    for (const f of this.files) {
-      let hitInFile = false;
-      for (const line of f.lines) {
-        if (!line.includes(query)) continue;
-        hitInFile = true;
-        // Heuristic: pull a likely caller name out of "function X(", "method X(",
-        // "const X =", "X(" near the beginning, etc. Rough but it's what a
-        // human-style grep+eyeball would produce.
-        const fnMatch =
-          line.match(/(?:function|class|interface)\s+([A-Za-z_$][\w$]*)/) ||
-          line.match(/\b([A-Za-z_$][\w$]*)\s*[:=]\s*(?:async\s+)?(?:function|\()/);
-        if (fnMatch) suspectedNames.add(fnMatch[1]);
-      }
-      if (hitInFile) {
-        matchedFiles.add(f.rel);
-        bytesRead += f.bytes;
-      }
-    }
-
-    return { matchedFiles, bytesRead, suspectedNames };
-  }
-
-  run(task: Task): RunResult {
-    const start = Date.now();
-    let returned: string[] = [];
-    let bytesRead = 0;
-
-    switch (task.kind) {
-      case 'callers':
-      case 'impact':
-      case 'recall':
-      case 'recall-substring':
-      case 'recall-miss': {
-        // Grep for the query string, then collect identifier names near
-        // each match. Best-effort approximation of what an agent would
-        // do without structural memory.
-        const { bytesRead: br, suspectedNames } = this.grepScan(task.query);
-        bytesRead = br;
-        // Exclude the query itself if it appears as a suspected name
-        // (the function declaration line will have the query as its own
-        // name; that's not a caller).
-        suspectedNames.delete(task.query);
-        returned = [...suspectedNames].sort();
-        break;
-      }
-
-      case 'kind-filter': {
-        // The query is a TypeScript keyword like "interface". Grep for
-        // "interface " (with trailing space) to filter declarations from
-        // string literals, then extract the next identifier.
-        const re = new RegExp(`\\b${task.query}\\s+([A-Za-z_$][\\w$]*)`, 'g');
-        const names = new Set<string>();
-        for (const f of this.files) {
-          if (!f.rel.startsWith('src/')) continue;
-          const text = f.lines.join('\n');
-          let m: RegExpExecArray | null;
-          let matched = false;
-          while ((m = re.exec(text)) !== null) {
-            names.add(m[1]);
-            matched = true;
-          }
-          if (matched) bytesRead += f.bytes;
-        }
-        returned = [...names].sort();
-        break;
-      }
-
-      case 'tests-affected': {
-        // Without structural memory, you'd grep for the symbol and look
-        // at which *.test.ts files contain it.
-        const { bytesRead: br, matchedFiles } = this.grepScan(task.query);
-        bytesRead = br;
-        returned = [...matchedFiles].filter((f) => /\.(test|spec)\.[jt]sx?$/.test(f)).sort();
-        break;
-      }
-
-      case 'string-literal': {
-        // GREP-NATIVE: just return matching files
-        const { bytesRead: br, matchedFiles } = this.grepScan(task.query);
-        bytesRead = br;
-        returned = [...matchedFiles].sort();
-        break;
-      }
-    }
-
-    return {
-      returned,
-      outputBytes: bytesRead, // baseline cost = bytes the agent would read
-      durationMs: Date.now() - start,
-    };
-  }
-}

package/bench/runner-graphpilot.ts

@@ -1,131 +0,0 @@
-/**
- * Run each benchmark task using GraphPilot's primitives directly.
- * Measures correctness against task.groundTruth + bytes of output the
- * tool returned (proxy for the token cost an agent would pay).
- */
-
-import { loadGraph } from '../src/storage.js';
-import { GraphIndex } from '../src/query.js';
-import { analyzeImpact } from '../src/impact.js';
-import type { Task } from './tasks.js';
-
-export interface RunResult {
-  /** Strings the tool returned. For caller/impact tasks, caller names. */
-  returned: string[];
-  /** Bytes the tool's structured output occupies as JSON. */
-  outputBytes: number;
-  /** Wall-clock for the tool call. */
-  durationMs: number;
-}
-
-export class GraphpilotRunner {
-  private readonly idx: GraphIndex;
-
-  constructor(repoRoot: string) {
-    const g = loadGraph(repoRoot);
-    if (!g) {
-      throw new Error(`No graph found at ${repoRoot}. Run \`graphpilot index\` first.`);
-    }
-    this.idx = new GraphIndex(g);
-  }
-
-  run(task: Task): RunResult {
-    const start = Date.now();
-    let returned: string[];
-
-    switch (task.kind) {
-      case 'callers': {
-        const target = this.idx.resolveSymbol(task.query);
-        if (!target) {
-          returned = [];
-          break;
-        }
-        const edges = this.idx.callers(target.id);
-        const names = new Set<string>();
-        for (const e of edges) {
-          const from = this.idx.findById(e.fromId);
-          if (from) names.add(from.name);
-        }
-        returned = [...names].sort();
-        break;
-      }
-
-      case 'recall': {
-        const matches = this.idx.findByName(task.query, { limit: 50 });
-        returned = matches.map((s) => s.name).sort();
-        break;
-      }
-
-      case 'recall-substring': {
-        const matches = this.idx.findByName(task.query, {
-          substring: true,
-          limit: 100,
-        });
-        returned = matches.map((s) => s.name).sort();
-        break;
-      }
-
-      case 'kind-filter': {
-        // Filter the full symbol table by kind. The MCP surface doesn't
-        // expose this as a tool today (would be a v0.2 gp_list_by_kind)
-        // but the data is in GraphIndex.graph.symbols.
-        returned = this.idx.graph.symbols
-          .filter((s) => s.kind === task.query && s.file.startsWith('src/'))
-          .map((s) => s.name)
-          .sort();
-        break;
-      }
-
-      case 'impact': {
-        const report = analyzeImpact(this.idx, task.query, { depth: 2 });
-        if (!report) {
-          returned = [];
-          break;
-        }
-        const names = new Set<string>();
-        for (const c of report.directCallers) names.add(c.symbol.name);
-        for (const c of report.transitiveCallers) names.add(c.symbol.name);
-        returned = [...names].sort();
-        break;
-      }
-
-      case 'tests-affected': {
-        const report = analyzeImpact(this.idx, task.query, { depth: 3 });
-        if (!report) {
-          returned = [];
-          break;
-        }
-        const files = new Set<string>();
-        for (const c of report.testsAffected) files.add(c.symbol.file);
-        returned = [...files].sort();
-        break;
-      }
-
-      case 'recall-miss': {
-        const matches = this.idx.findByName(task.query, { limit: 10 });
-        returned = matches.map((s) => s.name).sort();
-        break;
-      }
-
-      case 'string-literal': {
-        // GraphPilot intentionally doesn't index string literals or
-        // identifier usages outside structural contexts. Best effort:
-        // return any file where a symbol whose NAME matches the query
-        // is defined. This will miss most usages (the honest "grep wins"
-        // baseline).
-        const decl = this.idx.findByName(task.query, { limit: 5 });
-        const files = new Set<string>();
-        for (const s of decl) files.add(s.file);
-        returned = [...files].sort();
-        break;
-      }
-    }
-
-    const outputBytes = Buffer.byteLength(JSON.stringify(returned), 'utf8');
-    return {
-      returned,
-      outputBytes,
-      durationMs: Date.now() - start,
-    };
-  }
-}

package/bench/score-agent-tier.ts

@@ -1,191 +0,0 @@
-/**
- * Tier-B Agent Benchmark Scorer
- *
- * Parses Claude Code session transcripts (.jsonl) and produces the per-task
- * metrics table for the agent-eval benchmark.
- *
- * Usage:
- *   npx tsx bench/score-agent-tier.ts \
- *     --baseline <path-to-baseline.jsonl> \
- *     --graphpilot <path-to-graphpilot.jsonl> \
- *     --output bench/results/agent-tier-2026-05-22.md
- *
- * Input: Claude Code session jsonl (one JSON message per line)
- * Output: Markdown table with per-task metrics + aggregate stats
- */
-
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import { createReadStream } from 'node:fs';
-import { createInterface } from 'node:readline';
-
-interface Message {
-  type: string;
-  role?: string;
-  content?: string;
-  [key: string]: any;
-}
-
-interface TaskResult {
-  id: string;
-  prompt: string;
-  // Scorer fills in:
-  taskSuccessBaseline: 0 | 1;
-  taskSuccessGraphPilot: 0 | 1;
-  hallucCountBaseline: number;
-  hallucCountGraphPilot: number;
-  tokenCostBaseline: number;
-  tokenCostGraphPilot: number;
-  anchorResolutionRate: number; // 0..1, only for GP
-  diffNoiseRatio: number; // 0..1, only for GP (impact-since tasks)
-}
-
-async function readJsonl(filePath: string): Promise<Message[]> {
-  const messages: Message[] = [];
-  const rl = createInterface({
-    input: createReadStream(filePath),
-    crlfDelay: Infinity,
-  });
-
-  for await (const line of rl) {
-    if (line.trim()) {
-      messages.push(JSON.parse(line));
-    }
-  }
-  return messages;
-}
-
-function extractTokenUsage(msgs: Message[]): number {
-  let total = 0;
-  for (const msg of msgs) {
-    if (msg.usage) {
-      total += (msg.usage.input_tokens ?? 0) + (msg.usage.output_tokens ?? 0);
-    }
-  }
-  return total;
-}
-
-/**
- * Extract file:line @ sha anchors from transcript text.
- * Pattern: src/foo.ts:42 @ ab12cd3
- */
-function extractAnchors(text: string): Array<{ file: string; line: number; sha: string }> {
-  const anchors: Array<{ file: string; line: number; sha: string }> = [];
-  const re = /(\S+\.(?:ts|tsx|js|jsx)):(\d+)(?:\s+@\s+([0-9a-f]{7}))?/g;
-  let m;
-  while ((m = re.exec(text))) {
-    anchors.push({
-      file: m[1],
-      line: parseInt(m[2], 10),
-      sha: m[3] ?? '',
-    });
-  }
-  return anchors;
-}
-
-/**
- * Stub scorer: human-in-the-loop.
- *
- * Real scoring requires:
- *   1. Human opens the transcript
- *   2. Manually reads the agent's final answer
- *   3. Compares to ground truth (from tasks.ts)
- *   4. Marks success/hallucination count
- *
- * This function prompts for input or reads from a pre-filled CSV.
- * For now, it's a template showing the metrics to collect.
- */
-async function scoreTask(
-  taskId: string,
-  baselineTranscript: string,
-  graphpilotTranscript: string,
-): Promise<TaskResult> {
-  // TODO: Implement human-in-the-loop or CSV reader
-  // For MVP, return a stub result
-  return {
-    id: taskId,
-    prompt: '(placeholder)',
-    taskSuccessBaseline: 0,
-    taskSuccessGraphPilot: 0,
-    hallucCountBaseline: 0,
-    hallucCountGraphPilot: 0,
-    tokenCostBaseline: 0,
-    tokenCostGraphPilot: 0,
-    anchorResolutionRate: 0,
-    diffNoiseRatio: 0,
-  };
-}
-
-/**
- * Format results as Markdown table suitable for README
- */
-function formatResultsTable(results: TaskResult[]): string {
-  let md = `# Tier-B Agent Benchmark Results\n\n`;
-  md += `| Task | Baseline Success | GP Success | Halluc (B) | Halluc (GP) | Tokens (B) | Tokens (GP) | Anchor % | Diff Noise |\n`;
-  md += `|---|---|---|---|---|---|---|---|---|\n`;
-
-  for (const r of results) {
-    md += `| ${r.id} | ${r.taskSuccessBaseline} | ${r.taskSuccessGraphPilot} | ${r.hallucCountBaseline} | ${r.hallucCountGraphPilot} | ${r.tokenCostBaseline} | ${r.tokenCostGraphPilot} | ${(r.anchorResolutionRate * 100).toFixed(0)}% | ${(r.diffNoiseRatio * 100).toFixed(0)}% |\n`;
-  }
-
-  // Aggregate stats
-  const totalSuccessB = results.reduce((n, r) => n + r.taskSuccessBaseline, 0);
-  const totalSuccessGP = results.reduce((n, r) => n + r.taskSuccessGraphPilot, 0);
-  const totalHallucB = results.reduce((n, r) => n + r.hallucCountBaseline, 0);
-  const totalHallucGP = results.reduce((n, r) => n + r.hallucCountGraphPilot, 0);
-  const totalTokensB = results.reduce((n, r) => n + r.tokenCostBaseline, 0);
-  const totalTokensGP = results.reduce((n, r) => n + r.tokenCostGraphPilot, 0);
-  const avgAnchorRes = results.reduce((n, r) => n + r.anchorResolutionRate, 0) / results.length;
-
-  md += `\n## Summary\n\n`;
-  md += `- **Baseline success rate:** ${totalSuccessB}/${results.length}\n`;
-  md += `- **GraphPilot success rate:** ${totalSuccessGP}/${results.length}\n`;
-  md += `- **Hallucinations (Baseline):** ${totalHallucB}\n`;
-  md += `- **Hallucinations (GraphPilot):** ${totalHallucGP}\n`;
-  md += `- **Token cost (Baseline):** ${totalTokensB}\n`;
-  md += `- **Token cost (GraphPilot):** ${totalTokensGP} (−${((1 - totalTokensGP / totalTokensB) * 100).toFixed(0)}%)\n`;
-  md += `- **Anchor resolution rate:** ${(avgAnchorRes * 100).toFixed(0)}%\n`;
-
-  return md;
-}
-
-async function main() {
-  const args = process.argv.slice(2);
-  const baselineArg =
-    args.find((a) => a.startsWith('--baseline='))?.split('=')[1] ||
-    args[args.indexOf('--baseline') + 1];
-  const graphpilotArg =
-    args.find((a) => a.startsWith('--graphpilot='))?.split('=')[1] ||
-    args[args.indexOf('--graphpilot') + 1];
-  const outputArg =
-    args.find((a) => a.startsWith('--output='))?.split('=')[1] ||
-    args[args.indexOf('--output') + 1];
-
-  if (!baselineArg || !graphpilotArg || !outputArg) {
-    console.error(
-      'Usage: npx tsx bench/score-agent-tier.ts --baseline <path> --graphpilot <path> --output <path>',
-    );
-    process.exit(1);
-  }
-
-  console.log('Loading transcripts...');
-  const baselineData = await readJsonl(baselineArg);
-  const graphpilotData = await readJsonl(graphpilotArg);
-
-  console.log(`Baseline: ${baselineData.length} messages`);
-  console.log(`GraphPilot: ${graphpilotData.length} messages`);
-
-  // TODO: Parse transcripts by task boundary, invoke scorer for each pair
-  console.log('\n[Stub] Scoring requires human review. Prepare scoring input CSV with columns:');
-  console.log('  task_id, baseline_success (0/1), gp_success (0/1), baseline_halluc, gp_halluc');
-  console.log('Then run: npx tsx bench/score-agent-tier.ts --scores <csv> --output <path>');
-
-  const results: TaskResult[] = [];
-  // Placeholder: results would be populated from CSV or human input
-
-  const output = formatResultsTable(results);
-  fs.writeFileSync(outputArg, output);
-  console.log(`Results written to ${outputArg}`);
-}
-
-main().catch(console.error);