@graphpilot-oss/graphpilot 0.0.1 → 0.1.0

package/bench/score.ts

@@ -1,59 +0,0 @@
-/**
- * Scoring: precision/recall/F1 of a returned set against ground truth.
- *
- * We score whether the set of names returned MATCHES the expected set,
- * not whether it's identical — partial credit is honest. F1 is the
- * primary metric. Per-task results show all three.
- */
-
-export interface Scored {
-  precision: number;
-  recall: number;
-  f1: number;
-  intersectionSize: number;
-  truePositives: string[];
-  falsePositives: string[];
-  falseNegatives: string[];
-}
-
-export function score(returned: string[], groundTruth: string[]): Scored {
-  const ret = new Set(returned);
-  const gt = new Set(groundTruth);
-
-  const tp: string[] = [];
-  for (const r of ret) if (gt.has(r)) tp.push(r);
-
-  const fp: string[] = [];
-  for (const r of ret) if (!gt.has(r)) fp.push(r);
-
-  const fn: string[] = [];
-  for (const g of gt) if (!ret.has(g)) fn.push(g);
-
-  // Edge case: empty ground truth + empty return = perfect.
-  // (The recall-miss task hits this.)
-  if (gt.size === 0 && ret.size === 0) {
-    return {
-      precision: 1,
-      recall: 1,
-      f1: 1,
-      intersectionSize: 0,
-      truePositives: [],
-      falsePositives: [],
-      falseNegatives: [],
-    };
-  }
-
-  const precision = ret.size === 0 ? 0 : tp.length / ret.size;
-  const recall = gt.size === 0 ? 0 : tp.length / gt.size;
-  const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
-
-  return {
-    precision,
-    recall,
-    f1,
-    intersectionSize: tp.length,
-    truePositives: tp.sort(),
-    falsePositives: fp.sort(),
-    falseNegatives: fn.sort(),
-  };
-}

package/bench/tasks.ts

@@ -1,236 +0,0 @@
-/**
- * The 10-task benchmark corpus. Hand-curated against graphpilot indexing
- * itself, so anyone can `git clone` + `pnpm install` + `pnpm bench` and
- * see the same numbers.
- *
- * Each task carries its OWN ground truth so this file is the single
- * source of truth for what "correct" means. Ground truth was computed
- * by probing the live index at the time this file was authored; if the
- * corpus repo (graphpilot itself) is materially edited, ground truth
- * needs to be refreshed (see bench/README.md §Refreshing).
- *
- * Mix of task types is deliberate:
- *  - 7 tasks where GraphPilot's structural index should win
- *  - 1 task that's roughly a tie (negative result)
- *  - 1 task where grep should outperform GraphPilot (string-literal
- *    search). Keeping this in the corpus is what makes the benchmark
- *    honest.
- */
-
-export type TaskKind =
-  | 'callers' // who calls X?
-  | 'recall' // find symbol by exact name
-  | 'recall-substring' // find symbols whose name contains a fragment
-  | 'kind-filter' // find all symbols of kind=...
-  | 'impact' // blast-radius analysis
-  | 'impact-since' // differential impact: callers in changed files since a ref
-  | 'tests-affected' // which tests depend on this symbol
-  | 'recall-miss' // symbol that doesn't exist
-  | 'string-literal'; // text-only search — grep should win
-
-export interface Task {
-  id: string;
-  description: string;
-  /** What a developer / agent would naturally ask. */
-  prompt: string;
-  kind: TaskKind;
-  /** Argument to the task's natural tool (graphpilot side). */
-  query: string;
-  /**
-   * Set of expected symbol names (sorted) that the correct answer must
-   * contain. For tests-affected tasks, this is the set of test file paths.
-   */
-  groundTruth: string[];
-  /** Which side we expect to win on F1 score. */
-  expectedWinner: 'graphpilot' | 'grep' | 'tie';
-  /** Helpful for the README/results: structural vs text-only. */
-  difficulty: 'low' | 'medium' | 'high';
-}
-
-export const TASKS: Task[] = [
-  {
-    id: 't01-callers-analyzeImpact',
-    description: 'Find every function that calls analyzeImpact',
-    prompt: 'In this repo, what functions call analyzeImpact?',
-    kind: 'callers',
-    query: 'analyzeImpact',
-    groundTruth: ['handleGpImpact'],
-    expectedWinner: 'graphpilot',
-    difficulty: 'low',
-  },
-  {
-    id: 't02-callers-extractSymbols',
-    description: 'Find every direct caller of extractSymbols',
-    prompt: 'Who calls extractSymbols in this codebase?',
-    kind: 'callers',
-    query: 'extractSymbols',
-    groundTruth: ['indexDirectory', 'applyUpdate', 'symbolsOf'],
-    expectedWinner: 'graphpilot',
-    difficulty: 'low',
-  },
-  {
-    id: 't03-callers-validateRootPath',
-    description: 'Find every direct caller of validateRootPath',
-    prompt: 'Where is validateRootPath used in the codebase? List every callsite.',
-    kind: 'callers',
-    query: 'validateRootPath',
-    // Note: GraphWatcher constructor calls validateRootPath; the SymbolRecord
-    // for that constructor has name="constructor" (not "GraphWatcher").
-    groundTruth: ['cmdIndex', 'main', 'handleGpIndex', 'constructor'],
-    expectedWinner: 'graphpilot',
-    difficulty: 'medium',
-  },
-  {
-    id: 't04-recall-substring-parse',
-    description: 'Find every symbol whose name contains "parse"',
-    prompt: 'List every function, class, or interface whose name contains "parse".',
-    kind: 'recall-substring',
-    query: 'parse',
-    groundTruth: ['ParsedFile', 'getParser', 'parseFile', 'parseSource', 'parseToken'],
-    expectedWinner: 'graphpilot',
-    difficulty: 'low',
-  },
-  {
-    id: 't05-kind-filter-interfaces',
-    description: 'Enumerate all TypeScript interfaces under src/',
-    prompt: 'List every TypeScript interface defined under src/.',
-    kind: 'kind-filter',
-    query: 'interface', // means: kind === "interface"
-    groundTruth: [
-      'CallEdge',
-      'EdgeQueryOptions',
-      'Graph',
-      'GpCallersArgs',
-      'GpImpactArgs',
-      'GpIndexArgs',
-      'GpRecallArgs',
-      'GpStatsArgs',
-      'ImpactCaller',
-      'ImpactOptions',
-      'ImpactResult',
-      'IndexOptions',
-      'IndexResult',
-      'InteractionEntry',
-      'ParsedFile',
-      'RawCall',
-      'RecallOptions',
-      'SecretPattern',
-      'SymbolRecord',
-      'ToolResult',
-      'UpdateResult',
-      'ValidationContext',
-      'WatcherOptions',
-    ],
-    expectedWinner: 'graphpilot',
-    difficulty: 'medium',
-  },
-  {
-    id: 't06-impact-extractSymbols-depth2',
-    description: 'Compute blast radius of changing extractSymbols (depth 2)',
-    prompt:
-      "If I change extractSymbols's signature, what functions will I need to update? Include indirect callers up to two hops.",
-    kind: 'impact',
-    query: 'extractSymbols',
-    // Direct callers + their direct callers
-    groundTruth: [
-      // d=1
-      'indexDirectory',
-      'applyUpdate',
-      'symbolsOf',
-      // d=2 — callers of the above
-      'cmdIndex',
-      'handleGpIndex',
-      'handleEvent',
-      // symbolsOf has no callers in production code; it's only in tests
-    ],
-    expectedWinner: 'graphpilot',
-    difficulty: 'high',
-  },
-  {
-    id: 't07-tests-affected-parseFile',
-    description: 'Identify test files that exercise parseFile (directly)',
-    prompt: 'If I change the behavior of parseFile, which test files are most likely to break?',
-    kind: 'tests-affected',
-    query: 'parseFile',
-    // The test file containing symbolsOf which calls parseFile
-    groundTruth: ['tests/symbols.test.ts'],
-    expectedWinner: 'graphpilot',
-    difficulty: 'medium',
-  },
-  {
-    id: 't08-recall-substring-args',
-    description: 'Find every MCP-tool input-args interface',
-    prompt: 'List every TypeScript type whose name ends with "Args".',
-    kind: 'recall-substring',
-    query: 'Args',
-    groundTruth: ['GpCallersArgs', 'GpImpactArgs', 'GpIndexArgs', 'GpRecallArgs', 'GpStatsArgs'],
-    expectedWinner: 'graphpilot',
-    difficulty: 'low',
-  },
-  {
-    id: 't09-recall-miss',
-    description: 'Look up a symbol that does not exist (negative test)',
-    prompt: 'Find the function definitelyNotARealSymbol in this codebase.',
-    kind: 'recall-miss',
-    query: 'definitelyNotARealSymbol',
-    groundTruth: [], // empty set
-    expectedWinner: 'tie',
-    difficulty: 'low',
-  },
-  {
-    id: 't10-string-literal-MAX_FILE_BYTES',
-    description: 'Find every literal occurrence of the constant name "MAX_FILE_BYTES"',
-    prompt: 'Find every place the string "MAX_FILE_BYTES" appears in the source.',
-    kind: 'string-literal',
-    query: 'MAX_FILE_BYTES',
-    // We don't index string literals or identifier usages outside structural
-    // contexts — but for THIS specific constant the structural index has the
-    // declaration. Both should find the declaration; only grep finds every
-    // usage. We expect grep to win on recall here.
-    groundTruth: [
-      'src/validation.ts', // declared here
-      'src/parser.ts', // imported and used
-      'tests/security.test.ts', // referenced in a test
-    ],
-    expectedWinner: 'grep',
-    difficulty: 'medium',
-  },
-  {
-    id: 't11-impact-since-indexDirectory',
-    description: 'Differential impact: callers of indexDirectory changed since HEAD~1',
-    prompt:
-      'Show me callers of indexDirectory, but only those in files that have changed since HEAD~1. This is for PR review — I want to know which of my changes will be affected.',
-    kind: 'impact-since',
-    query: 'indexDirectory',
-    // Ground truth: callers of indexDirectory at all depths are
-    // [cmdIndex, handleGpIndex, applyUpdate]. On a clean repo HEAD~1
-    // should be empty or shallow-history, so we expect ~0 to all three
-    // depending on the branch state. Scorer will filter by changed files.
-    groundTruth: [], // Filled in during scoring based on actual git state
-    expectedWinner: 'graphpilot', // GraphPilot filters noise; baseline can't
-    difficulty: 'high',
-  },
-  {
-    id: 't12-evidence-anchor-resolution',
-    description: 'Evidence anchors: every tool response carries file:line @ sha citations',
-    prompt:
-      'Find every function that calls analyzeImpact. For each result, I need the exact file and line number I can jump to. Include the git SHA from when the index was built.',
-    kind: 'callers', // same tool as t01, different validation
-    query: 'analyzeImpact',
-    groundTruth: ['handleGpImpact'],
-    expectedWinner: 'graphpilot', // Only GP returns structured evidence anchors
-    difficulty: 'medium',
-  },
-  {
-    id: 't13-recall-nonexistent-with-anchor',
-    description:
-      'Anti-hallucination: looking up a symbol that does not exist returns citation proof',
-    prompt:
-      'Find the function fakeSymbolXYZ123. If it does not exist, show me the evidence — what query returned no results and why?',
-    kind: 'recall-miss',
-    query: 'fakeSymbolXYZ123',
-    groundTruth: [], // Does not exist
-    expectedWinner: 'graphpilot', // GP can cite "not in index" with proof; baseline may hallucinate
-    difficulty: 'high',
-  },
-];

package/dist/provenance.d.ts

@@ -1,74 +0,0 @@
-/**
- * Provenance / evidence anchors for tool responses.
- *
- * Why this exists: agents hallucinate. The most common Cursor / Claude
- * Code failure pattern is "the model called a function that doesn't
- * exist" — and the user has no quick way to verify a tool's claim
- * before acting on it.
- *
- * Solution: every result we return carries an evidence anchor — a
- * structured `{file, line, sha, excerpt}` reference that the agent
- * can include verbatim in its reply. The user (or the agent itself)
- * can then jump to the file/line and confirm the cited code matches
- * the claim. If we ever fabricate, the anchor exposes us instantly.
- *
- * Format design:
- *   - Path is relative to the indexed root (portable).
- *   - Line is 1-indexed (matches editor convention).
- *   - SHA is optional — null when the indexed repo isn't a git repo.
- *     When present, it's the 7-char short SHA of the index time.
- *   - Excerpt is optional and short (~200 chars) — for symbol records
- *     it's the signature line. For edges it's the call expression.
- *
- * Per the differentiation research, this is the SINGLE feature no
- * competitor in the 15+ landscape has shipped. See
- * .notes/differentiation-research-2026-05-21.md §1.3.
- */
-import type { SymbolRecord } from './symbols.js';
-import type { CallEdge } from './edges.js';
-/** A single evidence anchor pointing at a specific location in the repo. */
-export interface Provenance {
-    /** Relative path from the indexed repo root, e.g. "src/auth.ts". */
-    file: string;
-    /** 1-indexed line number. */
-    line: number;
-    /** Optional 1-indexed column. */
-    column?: number;
-    /** End line, when the entity spans multiple lines. */
-    endLine?: number;
-    /** Short git SHA (7 chars) at index time. Null if not a git repo. */
-    sha?: string | null;
-    /** Short text excerpt — symbol signature, call expression, etc. */
-    excerpt?: string;
-}
-/**
- * Make provenance for a symbol. Excerpt is the symbol's stored
- * signature (already secret-redacted by T3).
- */
-export declare function symbolProvenance(s: SymbolRecord, sha: string | null): Provenance;
-/**
- * Make provenance for a call edge. Points at the CALL SITE (where the
- * call happens), not the callee's definition. The callee's own
- * provenance is available via `symbolProvenance(idx.findById(edge.toId))`
- * when toId is non-null.
- */
-export declare function edgeProvenance(e: CallEdge, sha: string | null): Provenance;
-/**
- * Format a Provenance as a single-line human/agent-readable string.
- * Used inline in tool text responses.
- *
- * Examples:
- *   src/auth.ts:42                    (no sha — not a git repo)
- *   src/auth.ts:42 @ ab12cd3          (with sha)
- *   src/auth.ts:42:5 @ ab12cd3        (with column)
- */
-export declare function formatProvenance(p: Provenance): string;
-/**
- * Format a Provenance as a verifiable evidence tag the agent can
- * include in its reply. The agent's user (or a downstream tool) can
- * paste this directly into a search.
- *
- * Example:
- *   [evidence: src/auth.ts:42 @ ab12cd3]
- */
-export declare function formatEvidenceTag(p: Provenance): string;

package/dist/provenance.js

@@ -1,95 +0,0 @@
-/**
- * Provenance / evidence anchors for tool responses.
- *
- * Why this exists: agents hallucinate. The most common Cursor / Claude
- * Code failure pattern is "the model called a function that doesn't
- * exist" — and the user has no quick way to verify a tool's claim
- * before acting on it.
- *
- * Solution: every result we return carries an evidence anchor — a
- * structured `{file, line, sha, excerpt}` reference that the agent
- * can include verbatim in its reply. The user (or the agent itself)
- * can then jump to the file/line and confirm the cited code matches
- * the claim. If we ever fabricate, the anchor exposes us instantly.
- *
- * Format design:
- *   - Path is relative to the indexed root (portable).
- *   - Line is 1-indexed (matches editor convention).
- *   - SHA is optional — null when the indexed repo isn't a git repo.
- *     When present, it's the 7-char short SHA of the index time.
- *   - Excerpt is optional and short (~200 chars) — for symbol records
- *     it's the signature line. For edges it's the call expression.
- *
- * Per the differentiation research, this is the SINGLE feature no
- * competitor in the 15+ landscape has shipped. See
- * .notes/differentiation-research-2026-05-21.md §1.3.
- */
-const MAX_EXCERPT_LEN = 200;
-function clipExcerpt(s) {
-    if (!s)
-        return undefined;
-    const trimmed = s.trim();
-    if (trimmed.length === 0)
-        return undefined;
-    return trimmed.length > MAX_EXCERPT_LEN ? trimmed.slice(0, MAX_EXCERPT_LEN - 1) + '…' : trimmed;
-}
-/**
- * Make provenance for a symbol. Excerpt is the symbol's stored
- * signature (already secret-redacted by T3).
- */
-export function symbolProvenance(s, sha) {
-    return {
-        file: s.file,
-        line: s.line,
-        column: s.column,
-        endLine: s.endLine,
-        sha: sha ?? null,
-        excerpt: clipExcerpt(s.signature),
-    };
-}
-/**
- * Make provenance for a call edge. Points at the CALL SITE (where the
- * call happens), not the callee's definition. The callee's own
- * provenance is available via `symbolProvenance(idx.findById(edge.toId))`
- * when toId is non-null.
- */
-export function edgeProvenance(e, sha) {
-    return {
-        file: e.file,
-        line: e.line,
-        column: e.column,
-        sha: sha ?? null,
-        // No excerpt for edges — we don't store the call line text.
-        // (Future v0.2: capture the call line at index time so the agent
-        // sees the actual call expression.)
-    };
-}
-/**
- * Format a Provenance as a single-line human/agent-readable string.
- * Used inline in tool text responses.
- *
- * Examples:
- *   src/auth.ts:42                    (no sha — not a git repo)
- *   src/auth.ts:42 @ ab12cd3          (with sha)
- *   src/auth.ts:42:5 @ ab12cd3        (with column)
- */
-export function formatProvenance(p) {
-    let out = p.file + ':' + p.line;
-    if (p.column !== undefined)
-        out += ':' + p.column;
-    if (p.sha)
-        out += ' @ ' + p.sha;
-    return out;
-}
-/**
- * Format a Provenance as a verifiable evidence tag the agent can
- * include in its reply. The agent's user (or a downstream tool) can
- * paste this directly into a search.
- *
- * Example:
- *   [evidence: src/auth.ts:42 @ ab12cd3]
- */
-export function formatEvidenceTag(p) {
-    return `[evidence: ${formatProvenance(p)}]`;
-}
-//# sourceMappingURL=provenance.js.map

package/dist/provenance.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"provenance.js","sourceRoot":"","sources":["../src/provenance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAqBH,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,WAAW,CAAC,CAAqB;IACxC,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,OAAO,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;AAClG,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAe,EAAE,GAAkB;IAClE,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,GAAG,EAAE,GAAG,IAAI,IAAI;QAChB,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;KAClC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,CAAW,EAAE,GAAkB;IAC5D,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,EAAE,GAAG,IAAI,IAAI;QAChB,4DAA4D;QAC5D,iEAAiE;QACjE,oCAAoC;KACrC,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAa;IAC5C,IAAI,GAAG,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;IAChC,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,IAAI,CAAC,CAAC,GAAG;QAAE,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC;IAChC,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,CAAa;IAC7C,OAAO,cAAc,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9C,CAAC"}