@maxgfr/codeindex 2.17.1 → 2.18.0

package/src/engine-cli.ts

@@ -4,12 +4,14 @@ import { SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord } from "./types.js";
 import { ENGINE_VERSION } from "./types.js";
 import { ensureGrammars, grammarKeysForExts, resolveGrammarsTier, sharedGrammarsCacheDir } from "./ast/loader.js";
 import { resolveGrammarsPullTarget, pullGrammars } from "./ast/grammars-pull.js";
-import { buildIndexArtifacts, buildArtifactsFromScan, type BuildIndexOptions } from "./pipeline.js";
+import { buildIndexArtifacts, buildArtifactsFromScan, type BuildIndexOptions, type IndexArtifacts } from "./pipeline.js";
 import { sha1 } from "./hash.js";
 import { renderGraphJson } from "./render/graph-json.js";
 import { renderSymbolsJson } from "./render/symbols-json.js";
 import { renderScip } from "./render/scip.js";
-import { scanRepo } from "./scan.js";
+import { scanRepo, scanSummary, type RepoScan } from "./scan.js";
+import { scanRepoParallel } from "./pool.js";
+import { preloadSession, INDEX_DIR } from "./preload.js";
 import { walk, type WalkResult } from "./walk.js";
 import { buildCallerIndex } from "./callers.js";
 import { detectWorkspaces } from "./workspaces.js";
@@ -20,6 +22,8 @@ import { renderRepoMap } from "./repomap.js";
 import { findDeadCode } from "./deadcode.js";
 import { symbolComplexity, riskHotspots } from "./complexity.js";
 import { renderMermaid } from "./viz.js";
+import { impactOf, neighborsOf } from "./traverse.js";
+import { deltaFor, formatDeltaPanel } from "./delta.js";
 import { searchIndex } from "./bm25.js";
 import { checkRules, parseRules } from "./rules.js";
 import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, parseEmbedModel, resolveEmbedPullUrl, fetchEmbedModel } from "./embed/model.js";
@@ -83,6 +87,13 @@ Commands:
   complexity  Cyclomatic-complexity estimates, most-complex first. Pass a file
               positional for one file; omit for the repo-wide top
   risk        Complexity × git-churn ranking (JSON; --since <ref> to bound)
+  delta       Review panel for the git diff: changed files -> enclosing symbols ->
+              blast radius -> risk score with explained reasons
+              (--base <ref> | --staged, --depth <n>, --json)
+  impact      Reverse dependency closure of a file or module: everything that
+              transitively imports/uses/calls it (--depth <n>; JSON)
+  neighbors   Graph neighbours of a file or module, both directions
+              (--depth <n>, --kind import,call,use,doc-link,mention; JSON)
   mermaid     Mermaid diagram of the module graph; pass a module positional to
               focus on one neighborhood
   rewrite     Map an expensive tree-wide search onto its indexed equivalent:
@@ -97,7 +108,10 @@ Commands:
               check_rules, the memory quartet and the three symbolic-edit
               writes). Flags: --repo <dir> pins ONE repository so the per-tool
               repo argument becomes optional (an explicit per-call repo still
-              wins); --server-name <name> overrides the announced serverInfo
+              wins); --server-name <name> overrides the announced serverInfo;
+              --max-response-bytes <n> caps a single tool response (default 1e6;
+              a response under the cap is byte-identical, one over it is
+              replaced by an actionable notice instead of an unusable blob)
   version     Print the engine version
 
 Flags (accepted before OR after the subcommand: '--repo X scan' and
@@ -117,6 +131,16 @@ Flags (accepted before OR after the subcommand: '--repo X scan' and
   --max-bytes <n>     Skip files above this size (default 1 MiB)
   --max-calls <n>     Per-file call-site cap for extraction (default 512)
   --no-ast            Skip tree-sitter grammars even when present (regex tier)
+  --workers <n>       \`index\`: extraction worker threads (default: cores-1,
+                      capped at 8; 0 or 1 forces the single-threaded path).
+                      Also settable with CODEINDEX_WORKERS. Artifacts are
+                      byte-identical either way
+  --index <dir>       Persisted index the READ commands reuse, relative to the
+                      repo (default .codeindex — i.e. what \`index --out\` wrote
+                      there). A fresh index turns the scan into a stat pass and,
+                      when it still matches the worktree, skips the pipeline
+                      entirely. Stale/absent/corrupt → a normal cold build
+  --no-index-cache    Never reuse a persisted index; always build from scratch
   --config <file>     Rules config for \`rules\` (JSON: [{name, from, to, …}])
   --limit <n>         Max results for \`search\` (default 20)
   --no-fuzzy          \`search\`: disable trigram fuzzy fallback for query terms
@@ -144,6 +168,9 @@ interface CliFlags {
   maxBytes?: number;
   maxCalls?: number;
   noAst: boolean;
+  workers?: number; // extraction worker threads (0/1 = sequential)
+  indexDir?: string; // persisted index to read (default .codeindex)
+  noIndexCache?: boolean; // never reuse a persisted index
   since?: string;
   ignoreCase?: boolean;
   maxHits?: number;
@@ -155,6 +182,11 @@ interface CliFlags {
   recall?: boolean; // callers: recall-oriented binding
   run?: boolean; // `embed serve`: actually run the docker command (default: print)
   projectRoot?: string; // scip: override Metadata.project_root
+  base?: string; // delta: branch/ref to diff against (default: the repo's default branch)
+  staged?: boolean; // delta: diff the index instead of the merge-base
+  depth?: number; // delta/impact/neighbors: traversal hops
+  kind?: string; // neighbors: comma-separated edge kinds to traverse
+  json?: boolean; // delta: emit JSON instead of the human panel
   positional?: string; // e.g. the grep pattern or search query
 }
 
@@ -190,6 +222,15 @@ function parseFlags(args: string[]): CliFlags {
     else if (a === "--max-hits") flags.maxHits = num();
     else if (a === "--budget-tokens") flags.budgetTokens = num();
     else if (a === "--no-ast") flags.noAst = true;
+    else if (a === "--index") flags.indexDir = next();
+    else if (a === "--no-index-cache") flags.noIndexCache = true;
+    else if (a === "--workers") {
+      // 0 is meaningful here (force sequential), so this cannot use num().
+      const raw = next();
+      const n = Number(raw);
+      if (!Number.isInteger(n) || n < 0) throw new Error(`--workers expects a non-negative integer, got "${raw}"`);
+      flags.workers = n;
+    }
     else if (a === "--since") flags.since = next();
     else if (a === "--config") flags.config = resolve(next());
     else if (a === "--limit") flags.limit = num();
@@ -197,6 +238,11 @@ function parseFlags(args: string[]): CliFlags {
     else if (a === "--semantic") flags.semantic = true;
     else if (a === "--recall") flags.recall = true;
     else if (a === "--run") flags.run = true;
+    else if (a === "--base") flags.base = next();
+    else if (a === "--staged") flags.staged = true;
+    else if (a === "--depth") flags.depth = num();
+    else if (a === "--kind") flags.kind = next();
+    else if (a === "--json") flags.json = true;
     else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a;
     else throw new Error(`unknown flag: ${a}`);
   }
@@ -238,9 +284,14 @@ const SCANLESS_COMMANDS = new Set(["grep", "churn", "coupling", "workspaces", "g
 // dispatch site). `--repo` is resolved to an absolute path and must exist: a
 // server pinned to a typo'd directory would otherwise answer every tool call
 // with the same confusing per-call error instead of failing at startup.
-export function parseMcpFlags(argv: string[]): { defaultRepo?: string; serverInfo?: { name?: string } } {
+export function parseMcpFlags(argv: string[]): {
+  defaultRepo?: string;
+  serverInfo?: { name?: string };
+  maxResponseBytes?: number;
+} {
   let defaultRepo: string | undefined;
   let name: string | undefined;
+  let maxResponseBytes: number | undefined;
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     if (a === "--repo") {
@@ -251,12 +302,17 @@ export function parseMcpFlags(argv: string[]): { defaultRepo?: string; serverInf
       const v = argv[++i];
       if (!v) throw new Error("--server-name requires a value");
       name = v;
+    } else if (a === "--max-response-bytes") {
+      const v = argv[++i];
+      const n = Number(v);
+      if (!v || !Number.isFinite(n) || n <= 0) throw new Error("--max-response-bytes requires a positive number");
+      maxResponseBytes = n;
     } else {
       throw new Error(`unknown flag for \`mcp\`: ${a}`);
     }
   }
   if (defaultRepo && !existsSync(defaultRepo)) throw new Error(`--repo path does not exist: ${defaultRepo}`);
-  return { defaultRepo, serverInfo: name ? { name } : undefined };
+  return { defaultRepo, serverInfo: name ? { name } : undefined, maxResponseBytes };
 }
 
 // Flags that consume the following argv element. Needed to hoist leading flags
@@ -279,6 +335,9 @@ const VALUE_FLAGS = new Set([
   "--config",
   "--limit",
   "--server-name",
+  "--workers",
+  "--index",
+  "--max-response-bytes",
 ]);
 
 // Accept global flags BEFORE the subcommand as well as after, so
@@ -369,6 +428,38 @@ export async function runCli(rawArgv: string[]): Promise<void> {
     await ensureGrammars(grammarKeysForExts(precomputedWalk.files.map((f) => f.ext)));
   }
 
+  // Read commands reuse a persisted index instead of rebuilding from scratch.
+  //
+  // Only `index` ever consulted .codeindex/; every read command (graph, symbols,
+  // scip, callers, search, repomap, hotspots, deadcode, complexity, risk,
+  // mermaid, rules) re-walked, re-read, re-hashed and re-EXTRACTED the whole
+  // repo on each invocation — `codeindex search` cost a full tree-sitter pass
+  // every time, with a fresh index sitting right next to it.
+  //
+  // cache.json turns the scan into a stat pass; when the freshness guard holds,
+  // graph.json/symbols.json come back without running the pipeline at all. Both
+  // degrade to today's cold path when the index is absent, stale or corrupt, so
+  // output is unchanged either way. Resolved lazily and at most once: a command
+  // uses either the scan or the artifacts, never both.
+  const indexDir = flags.indexDir ?? INDEX_DIR;
+  let preloadTried = false;
+  let preloaded: { scan: RepoScan; arts?: IndexArtifacts } | undefined;
+  const tryPreload = (): { scan: RepoScan; arts?: IndexArtifacts } | undefined => {
+    if (preloadTried) return preloaded;
+    preloadTried = true;
+    if (flags.noIndexCache) return undefined;
+    const p = preloadSession(flags.repo, scanOptions(flags, precomputedWalk), indexDir);
+    if (p) preloaded = { scan: p.scan, arts: p.arts };
+    return preloaded;
+  };
+  const readScan = (): RepoScan => tryPreload()?.scan ?? scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+  const readArtifacts = (): IndexArtifacts => {
+    const p = tryPreload();
+    if (p?.arts) return p.arts;
+    if (p) return buildArtifactsFromScan(p.scan, scanOptions(flags, precomputedWalk));
+    return buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+  };
+
   if (cmd === "index") {
     if (!flags.out) throw new Error("index needs --out <dir>");
     const outDir = flags.out;
@@ -410,7 +501,12 @@ export async function runCli(rawArgv: string[]): Promise<void> {
     } catch {
       // no cache yet (or unreadable) — cold build
     }
-    const scan = scanRepo(flags.repo, { ...scanOptions(flags, precomputedWalk), cache, out: outDir });
+    const scan = await scanRepoParallel(flags.repo, {
+      ...scanOptions(flags, precomputedWalk),
+      cache,
+      out: outDir,
+      workers: flags.workers,
+    });
     const modelDir = resolveEmbedModelDir(flags.repo);
     const model = modelDir ? loadEmbedModel(modelDir) : undefined;
 
@@ -512,23 +608,26 @@ export async function runCli(rawArgv: string[]): Promise<void> {
       process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
     }
   } else if (cmd === "scan") {
-    const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+    // Summary-only: a file count and a language histogram need the walk and the
+    // path-based classifiers, never a read or a parse. Same numbers as before by
+    // construction — scanSummary and scanRepo share the keptFiles loop.
+    const s = scanSummary(flags.repo, scanOptions(flags, precomputedWalk));
     const summary = {
       engineVersion: ENGINE_VERSION,
-      commit: scan.commit,
-      fileCount: scan.files.length,
-      languages: scan.languages,
-      capped: scan.capped,
+      commit: s.commit,
+      fileCount: s.fileCount,
+      languages: s.languages,
+      capped: s.capped,
     };
     emit(JSON.stringify(summary, null, 2) + "\n", flags.out);
   } else if (cmd === "graph") {
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+    const { graph } = readArtifacts();
     emit(renderGraphJson(graph), flags.out);
   } else if (cmd === "symbols") {
-    const { symbols } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+    const { symbols } = readArtifacts();
     emit(renderSymbolsJson(symbols), flags.out);
   } else if (cmd === "scip") {
-    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+    const scan = readScan();
     const bytes = renderScip(scan, { projectRoot: flags.projectRoot });
     const out = flags.out ?? resolve("index.scip");
     if (out === "-") process.stdout.write(Buffer.from(bytes));
@@ -537,14 +636,14 @@ export async function runCli(rawArgv: string[]): Promise<void> {
       process.stderr.write(`codeindex: SCIP index → ${out} (${bytes.length} bytes)\n`);
     }
   } else if (cmd === "callers") {
-    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+    const scan = readScan();
     const index = buildCallerIndex(scan, undefined, { recall: flags.recall });
     const obj: Record<string, unknown> = {};
     for (const [name, entry] of index) obj[name] = entry;
     emit(JSON.stringify(obj, null, 2) + "\n", flags.out);
   } else if (cmd === "search") {
     if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
-    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+    const scan = readScan();
     if (flags.semantic) {
       const endpoint = resolveEmbedEndpoint();
       const lexical = (): void => {
@@ -649,7 +748,7 @@ export async function runCli(rawArgv: string[]): Promise<void> {
       }
       const model = loadEmbedModel(modelDir)!;
       mkdirSync(flags.out, { recursive: true });
-      const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+      const scan = readScan();
       const index = buildEmbeddingIndex(scan, model);
       writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index));
       process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`);
@@ -721,7 +820,7 @@ export async function runCli(rawArgv: string[]): Promise<void> {
   } else if (cmd === "rules") {
     if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
     const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8")));
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+    const { graph } = readArtifacts();
     const violations = checkRules(graph, rules);
     const errors = violations.filter((v) => v.severity === "error").length;
     emit(JSON.stringify({ errors, warnings: violations.length - errors, violations }, null, 2) + "\n", flags.out);
@@ -742,26 +841,48 @@ export async function runCli(rawArgv: string[]): Promise<void> {
     for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!;
     emit(JSON.stringify({ ok, churn: sorted }, null, 2) + "\n", flags.out);
   } else if (cmd === "repomap") {
-    const { scan, graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+    const { scan, graph } = readArtifacts();
     emit(renderRepoMap(scan, graph, { budgetTokens: flags.budgetTokens }), flags.out);
   } else if (cmd === "hotspots") {
-    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+    const scan = readScan();
     const { churn, ok } = gitChurn(flags.repo, { since: flags.since });
     emit(JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2) + "\n", flags.out);
   } else if (cmd === "coupling") {
     const { ok, couplings } = changeCoupling(flags.repo, { since: flags.since });
     emit(JSON.stringify({ ok, couplings }, null, 2) + "\n", flags.out);
   } else if (cmd === "deadcode") {
-    emit(JSON.stringify(findDeadCode(scanRepo(flags.repo, scanOptions(flags, precomputedWalk))), null, 2) + "\n", flags.out);
+    emit(JSON.stringify(findDeadCode(readScan()), null, 2) + "\n", flags.out);
   } else if (cmd === "complexity") {
-    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+    const scan = readScan();
     emit(JSON.stringify(symbolComplexity(scan, flags.positional), null, 2) + "\n", flags.out);
   } else if (cmd === "risk") {
-    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
+    const scan = readScan();
     const { churn, ok } = gitChurn(flags.repo, { since: flags.since });
     emit(JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2) + "\n", flags.out);
+  } else if (cmd === "delta") {
+    const { graph, symbols } = readArtifacts();
+    const res = deltaFor(flags.repo, graph, symbols, {
+      base: flags.base,
+      staged: flags.staged,
+      depth: flags.depth,
+    });
+    if ("error" in res) throw new Error(res.error);
+    emit(flags.json ? JSON.stringify(res, null, 2) + "\n" : formatDeltaPanel(res), flags.out);
+  } else if (cmd === "impact") {
+    if (!flags.positional) throw new Error("impact needs a target: cli.mjs impact <file|module> --repo <dir>");
+    const { graph } = readArtifacts();
+    const res = impactOf(graph, flags.positional, flags.depth ?? Infinity);
+    if (!res) throw new Error(`no such file or module in the index: ${flags.positional}`);
+    emit(JSON.stringify(res, null, 2) + "\n", flags.out);
+  } else if (cmd === "neighbors") {
+    if (!flags.positional) throw new Error("neighbors needs a target: cli.mjs neighbors <file|module> --repo <dir>");
+    const { graph } = readArtifacts();
+    const kinds = flags.kind ? new Set(flags.kind.split(",").map((k) => k.trim()).filter(Boolean)) : undefined;
+    const res = neighborsOf(graph, flags.positional, flags.depth ?? 1, kinds);
+    if (!res) throw new Error(`no such file or module in the index: ${flags.positional}`);
+    emit(JSON.stringify(res, null, 2) + "\n", flags.out);
   } else if (cmd === "mermaid") {
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
+    const { graph } = readArtifacts();
     emit(renderMermaid(graph, { module: flags.positional }), flags.out);
   } else if (cmd === "grep") {
     if (!flags.positional) throw new Error("grep needs a pattern: cli.mjs grep <pattern> --repo <dir>");

package/src/engine.ts

@@ -28,8 +28,23 @@ export type {
 // Files tier: walk, read, filter, classify.
 export { walk, readText, DEFAULT_MAX_FILES } from "./walk.js";
 export type { WalkOptions, WalkedFile, WalkResult } from "./walk.js";
-export { scanRepo } from "./scan.js";
-export type { RepoScan, ScanOptions } from "./scan.js";
+export { scanRepo, scanSummary } from "./scan.js";
+export type { RepoScan, ScanOptions, ScanSummary, ExtractedRecord } from "./scan.js";
+export { keptCodeFiles, buildCodeRecord } from "./scan.js";
+// Reusing a persisted `.codeindex/` index instead of rebuilding it. The MCP
+// server and every CLI read command go through this; a consumer that vendors
+// the engine gets the same shortcut. Every function degrades to undefined
+// (= "build it yourself") rather than throwing.
+export { preloadSession, preloadArtifacts, readPersistedIndex, toCacheMap, INDEX_DIR } from "./preload.js";
+export type { PersistedMeta, PersistedCacheEntry, PersistedCacheMap } from "./preload.js";
+// Parallel extraction. scanRepoParallel is scanRepo with the code files
+// extracted across worker_threads; it returns the same RepoScan, byte-for-byte,
+// and degrades to the sequential path whenever workers are unavailable.
+//
+// `runExtractWorker` MUST stay exported: the worker bootstrap imports it BY NAME
+// off this barrel, and a missing export is indistinguishable from "this is not
+// the engine" — the pool would silently run sequential forever.
+export { scanRepoParallel, extractInParallel, runExtractWorker, workerCount } from "./pool.js";
 export { compileGlobs } from "./glob.js";
 export { parseGitignore, isIgnored } from "./ignore.js";
 export type { IgnoreRule } from "./ignore.js";
@@ -177,8 +192,20 @@ export { findDeadCode } from "./deadcode.js";
 export type { DeadSymbol } from "./deadcode.js";
 export { symbolComplexity, riskHotspots, complexityOfSource } from "./complexity.js";
 export type { SymbolComplexity, RiskHotspot } from "./complexity.js";
-export { renderMermaid } from "./viz.js";
-export type { MermaidOptions } from "./viz.js";
+export { renderMermaid, renderMermaidClustered } from "./viz.js";
+export type { MermaidOptions, ClusteredMermaidOptions, ClusteredMermaidResult } from "./viz.js";
+
+// Graph traversal: reverse-dependency closure ("what breaks if I change this")
+// and bidirectional neighbourhood walks. Pure functions of a Graph, so a
+// consumer holding a persisted graph.json answers both without a rescan.
+export { impactOf, neighborsOf, reverseClosure, hubThreshold } from "./traverse.js";
+export type { ImpactResult, ImpactedFile, NeighborResult, NeighborLink } from "./traverse.js";
+
+// Diff review: git diff -> enclosing symbols -> blast radius -> risk-scored,
+// reasons-first panel. `computeDelta` is the pure core (no git, no fs);
+// `deltaFor` adds the git plumbing against a graph the caller supplies.
+export { computeDelta, deltaFor, formatDeltaPanel, symbolsInHunks, RISK_WEIGHTS, DEFAULT_DELTA_DEPTH } from "./delta.js";
+export type { DeltaOptions, DeltaResult, DeltaError, DeltaModule, DeltaChange, ChangedSymbol } from "./delta.js";
 export type { RepoMapOptions } from "./repomap.js";
 
 // MCP server over stdio (also reachable as `engine.mjs mcp`).

package/src/extract/code.ts

@@ -360,7 +360,10 @@ export function extractCode(rel: string, ext: string, content: string, opts: { m
   // extractors. Imports/pkg stay on the battle-tested regex path here — their
   // resolution is covered by resolve tests and the e2e ratchet; the new-language
   // AST importers land with their resolvers.
-  const ast = extractAst(rel, ext, content, { maxCalls: opts.maxCallsPerFile });
+  // `imports: false` because of exactly that: `ast.refs` and `ast.pkg` were
+  // computed by a full extra tree traversal and then discarded right below, in
+  // favour of the regex results. Public `extractAst` still computes them.
+  const ast = extractAst(rel, ext, content, { maxCalls: opts.maxCallsPerFile, imports: false });
   const symbols = (ast ? ast.symbols : extractSymbols(rel, ext, content)).slice(0, 400);
   // Add barrel re-exports the local def didn't already cover.
   const known = new Set(symbols.map((s) => s.name));