@maxgfr/codeindex 2.13.0 → 2.14.0

package/src/mcp.ts

@@ -8,11 +8,12 @@
 import { statSync } from "node:fs";
 import { join } from "node:path";
 import { createInterface } from "node:readline";
-import { ENGINE_VERSION } from "./types.js";
-import { ensureGrammars, allGrammarKeys } from "./ast/loader.js";
-import { buildIndexArtifacts } from "./pipeline.js";
+import { ENGINE_VERSION, type FileRecord } from "./types.js";
+import { ensureGrammars, grammarKeysForExts } from "./ast/loader.js";
+import { buildArtifactsFromScan, type IndexArtifacts } from "./pipeline.js";
 import { renderGraphJson } from "./render/graph-json.js";
-import { scanRepo, type RepoScan } from "./scan.js";
+import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js";
+import { walk } from "./walk.js";
 import { buildCallerIndex } from "./callers.js";
 import { detectWorkspaces } from "./workspaces.js";
 import { gitChurn } from "./git.js";
@@ -393,13 +394,157 @@ export function memoizedEmbedModel(modelDir: string): StaticEmbedModel | undefin
   return model;
 }
 
+// --- session-level scan + artifacts memoization ------------------------------
+// Same single-entry discipline as the embedding caches above: the MCP server
+// process is long-lived, but every tool call used to redo a FULL scanRepo walk
+// + read + hash + extraction pass (and, for graph-shaped tools, the whole
+// pipeline) even when nothing in the repo changed between requests. Cache the
+// last (repo, scan-opts) scan and feed its records back to scanRepo as `cache`
+// on the next call — scan.ts's EXISTING stat-fastpath + exact-hash machinery
+// is the freshness oracle, so a cache hit costs one walk + per-file stats, not
+// reads. When the oracle proves the content unchanged the SAME RepoScan object
+// is returned, which keeps the per-scan WeakMap of derived structures
+// (src/derived.ts) warm across requests. Artifacts are memoized on scan object
+// identity. Rendered strings are NEVER memoized — a big repo's graph.json runs
+// tens of MB, so renders stay per-call while the expensive structures behind
+// them are reused.
+//
+// Determinism: reused records come from scan.ts's own reuse paths (stat
+// fastpath / exact content-hash match), which produce records value-identical
+// to a from-scratch scan — artifacts stay byte-identical; only repeated work
+// disappears.
+
+// The scan options a session entry is keyed on. `cache`/`precomputedWalk` are
+// excluded from the contract: the session cache OWNS the cache it feeds back,
+// and a caller-supplied stale walk would desynchronize the freshness oracle.
+export type SessionScanOptions = Omit<ScanOptions, "cache" | "precomputedWalk">;
+
+type SessionCacheMap = Map<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }>;
+
+// A SINGLE entry — never an unbounded map — holding the most recent scan.
+let sessionCache:
+  | { key: string; scan: RepoScan; cacheMap: SessionCacheMap; arts?: IndexArtifacts }
+  | undefined;
+
+// Fixed property order (and JSON.stringify dropping undefined) keeps the key
+// deterministic regardless of how the caller assembled the options object.
+function sessionKey(repo: string, opts: SessionScanOptions): string {
+  return (
+    repo +
+    "\0" +
+    JSON.stringify({
+      scope: opts.scope,
+      include: opts.include,
+      exclude: opts.exclude,
+      gitignore: opts.gitignore,
+      ignoreDirs: opts.ignoreDirs,
+      maxBytes: opts.maxBytes,
+      maxFiles: opts.maxFiles,
+      maxCallsPerFile: opts.maxCallsPerFile,
+      out: opts.out,
+      fullHash: opts.fullHash,
+    })
+  );
+}
+
+// A scan re-expressed as the `ScanOptions.cache` shape (the exact map the CLI
+// persists as cache.json): rel → (hash, record, size, mtimeMs), so the next
+// scanRepo can take the stat fastpath / hash-match reuse paths against it.
+// Exported for tests.
+export function toCacheMap(scan: RepoScan): SessionCacheMap {
+  const m: SessionCacheMap = new Map();
+  for (const f of scan.files) m.set(f.rel, { hash: f.hash, record: f, size: f.size, mtimeMs: scan.mtimes.get(f.rel) });
+  return m;
+}
+
+// The memoizing replacement for scanRepo inside callTool. Exported for tests.
+export function getScan(repo: string, opts: SessionScanOptions = {}): RepoScan {
+  const key = sessionKey(repo, opts);
+  if (sessionCache && sessionCache.key === key) {
+    const fresh = scanRepo(repo, { ...opts, cache: sessionCache.cacheMap });
+    if (fresh.contentUnchanged) {
+      // Content proven identical → return the SAME object (object identity is
+      // what keeps derived.ts's WeakMap and the memoized artifacts warm). A
+      // stat-only drift (e.g. a bare touch) still refreshes the cache map so
+      // the next call's stat fastpath keys on the new (size, mtimeMs).
+      if (fresh.cacheDirty) sessionCache.cacheMap = toCacheMap(fresh);
+      // `commit` (headCommit(root)) is NOT part of the stat/hash freshness
+      // oracle: a git HEAD move that leaves the worktree untouched — commit /
+      // commit --amend / reset --soft / checkout to an identical-tree branch —
+      // changes headCommit without altering any file's size or mtime, so
+      // contentUnchanged stays true while the cached scan's commit went stale.
+      // `fresh` recomputed it just now (exactly what a cold process reports), so
+      // sync it onto the returned object; otherwise scan_summary would emit the
+      // OLD commit a from-scratch scanRepo never would. Mutate the SAME object
+      // rather than clone — cloning would forfeit the identity the artifacts and
+      // derived.ts WeakMap key on. Safe: no artifact carries commit (graph /
+      // symbols render byte-identically regardless), so nothing memoized here
+      // depends on this field.
+      if (sessionCache.scan.commit !== fresh.commit) sessionCache.scan.commit = fresh.commit;
+      return sessionCache.scan;
+    }
+    sessionCache = { key, scan: fresh, cacheMap: toCacheMap(fresh) };
+    return fresh;
+  }
+  const scan = scanRepo(repo, opts);
+  sessionCache = { key, scan, cacheMap: toCacheMap(scan) };
+  return scan;
+}
+
+// Lazy pipeline memoized on scan OBJECT IDENTITY: graph-shaped tools reuse the
+// artifacts exactly as long as getScan keeps returning the same scan object.
+// Exported for tests.
+export function getArtifacts(repo: string, opts: SessionScanOptions = {}): IndexArtifacts {
+  const scan = getScan(repo, opts);
+  if (sessionCache && sessionCache.scan === scan) {
+    return (sessionCache.arts ??= buildArtifactsFromScan(scan, opts));
+  }
+  // Defensive fallback (getScan always leaves sessionCache holding `scan`).
+  return buildArtifactsFromScan(scan, opts);
+}
+
+// Warm the grammars for the languages CURRENTLY present in `repo`, re-derived on
+// EVERY scan-needing call — never frozen on first touch. The server no longer
+// warms every committed grammar at startup; most sessions touch one repo and a
+// handful of languages, so each scan-needing tool warms the walk-derived set
+// itself. It MUST re-derive per call because the session cache (getScan) is built
+// to pick up mid-session file adds/edits/removes: a language whose first file
+// appears only AFTER the initial scan-needing call must still get its grammar
+// warmed, or that file falls to the regex tier and its symbols diverge from a
+// cold build on the identical on-disk state — a byte-identity break. (A per-
+// repo-path memo froze the grammar set at first touch and silently missed
+// exactly this case.) ensureGrammars is idempotent and near-free once a grammar
+// is loaded — it warms only newly-seen keys — so the sole repeated cost is the
+// walk; the wasm for a given language loads at most once. Determinism: the walk's
+// extension set is a superset of what scanRepo keeps (scope/include/exclude only
+// filter further), so every extracted file has its grammar loaded; Language.load
+// calls are independent, so warming fewer grammars cannot alter the parse of a
+// loaded one.
+export async function warmGrammarsForRepo(repo: string): Promise<void> {
+  const { files } = walk(repo, {});
+  await ensureGrammars(grammarKeysForExts(files.map((f) => f.ext)));
+}
+
+// Tools that never scan the file tree (git/grep/memory/embed-status only) — they
+// must not trigger a grammar warm. Every other tool is scan-needing and warms
+// the repo's grammars first; defaulting to "warm" keeps a newly added scan tool
+// correct without having to be listed here.
+const SCANLESS_TOOLS = new Set([
+  "workspaces", "churn", "coupling", "grep",
+  "write_memory", "read_memory", "list_memories", "delete_memory",
+  "embed_status",
+]);
+
 async function callTool(name: string, args: Record<string, unknown>): Promise<string> {
   const repo = str(args.repo);
   if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
   const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
+  // Scan-needing tools warm the present-language grammars (re-derived per call)
+  // before any scan so extraction takes the AST tier; scan-less tools skip it.
+  if (!SCANLESS_TOOLS.has(name)) await warmGrammarsForRepo(repo);
 
   if (name === "scan_summary") {
-    const scan = scanRepo(repo, scanOpts);
+    const scan = getScan(repo, scanOpts);
     return JSON.stringify(
       { engineVersion: ENGINE_VERSION, commit: scan.commit, fileCount: scan.files.length, languages: scan.languages, capped: scan.capped },
       null,
@@ -407,10 +552,10 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
     );
   }
   if (name === "graph") {
-    return renderGraphJson(buildIndexArtifacts(repo, scanOpts).graph);
+    return renderGraphJson(getArtifacts(repo, scanOpts).graph);
   }
   if (name === "symbols") {
-    const { symbols } = buildIndexArtifacts(repo, scanOpts);
+    const { symbols } = getArtifacts(repo, scanOpts);
     const lookup = str(args.name);
     if (lookup) {
       return JSON.stringify({ name: lookup, defs: symbols.defs[lookup] ?? [], refs: symbols.refs[lookup] ?? [] }, null, 2);
@@ -418,7 +563,7 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
     return JSON.stringify(symbols, null, 2);
   }
   if (name === "callers") {
-    const index = buildCallerIndex(scanRepo(repo, scanOpts));
+    const index = buildCallerIndex(getScan(repo, scanOpts));
     const lookup = str(args.name);
     if (lookup) {
       const entry = index.get(lookup);
@@ -441,12 +586,12 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
   if (name === "symbols_overview") {
     const file = str(args.file);
     if (!file) throw new Error("`file` is required");
-    return JSON.stringify(symbolsOverview(scanRepo(repo, scanOpts), file), null, 2);
+    return JSON.stringify(symbolsOverview(getScan(repo, scanOpts), file), null, 2);
   }
   if (name === "find_symbol") {
     const namePath = str(args.namePath);
     if (!namePath) throw new Error("`namePath` is required");
-    const matches = findSymbol(scanRepo(repo, scanOpts), namePath, {
+    const matches = findSymbol(getScan(repo, scanOpts), namePath, {
       substring: args.substring === true,
       includeBody: args.includeBody === true,
     });
@@ -455,15 +600,23 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
   if (name === "find_references") {
     const symName = str(args.name);
     if (!symName) throw new Error("`name` is required");
-    return JSON.stringify(findReferences(scanRepo(repo, scanOpts), symName), null, 2);
+    return JSON.stringify(findReferences(getScan(repo, scanOpts), symName), null, 2);
   }
   if (name === "replace_symbol_body" || name === "insert_after_symbol" || name === "insert_before_symbol") {
     const namePath = str(args.namePath);
     const body = typeof args.body === "string" ? args.body : undefined;
     if (!namePath || body === undefined) throw new Error("`namePath` and `body` are required");
-    const scan = scanRepo(repo, scanOpts);
+    const scan = getScan(repo, scanOpts);
     const fn = name === "replace_symbol_body" ? replaceSymbolBody : name === "insert_after_symbol" ? insertAfterSymbol : insertBeforeSymbol;
-    return JSON.stringify(fn(scan, namePath, body, str(args.file)), null, 2);
+    const result = fn(scan, namePath, body, str(args.file));
+    // A write WE just performed must not be trusted to the stat oracle: an
+    // edit landing in the same mtime tick with the same byte count would pass
+    // the (size, mtimeMs) fastpath and serve a stale scan. Drop the whole
+    // session entry unconditionally — the next call rescans from scratch.
+    // (write_memory needs no invalidation: .codeindex/ is excluded from the
+    // walk, so memories never enter a scan.)
+    sessionCache = undefined;
+    return JSON.stringify(result, null, 2);
   }
   if (name === "write_memory") {
     const memName = str(args.name);
@@ -487,10 +640,10 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
     return JSON.stringify({ deleted: deleteMemory(repo, memName) }, null, 2);
   }
   if (name === "dead_code") {
-    return JSON.stringify(findDeadCode(scanRepo(repo, scanOpts)), null, 2);
+    return JSON.stringify(findDeadCode(getScan(repo, scanOpts)), null, 2);
   }
   if (name === "complexity") {
-    const scan = scanRepo(repo, scanOpts);
+    const scan = getScan(repo, scanOpts);
     if (args.risk === true) {
       const { churn, ok } = gitChurn(repo);
       return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2);
@@ -498,15 +651,15 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
     return JSON.stringify(symbolComplexity(scan, str(args.file)), null, 2);
   }
   if (name === "mermaid") {
-    const { graph } = buildIndexArtifacts(repo, scanOpts);
+    const { graph } = getArtifacts(repo, scanOpts);
     return renderMermaid(graph, { module: str(args.module) });
   }
   if (name === "repo_map") {
-    const { scan, graph } = buildIndexArtifacts(repo, scanOpts);
+    const { scan, graph } = getArtifacts(repo, scanOpts);
     return renderRepoMap(scan, graph, { budgetTokens: typeof args.budgetTokens === "number" ? args.budgetTokens : undefined });
   }
   if (name === "hotspots") {
-    const scan = scanRepo(repo, scanOpts);
+    const scan = getScan(repo, scanOpts);
     const { churn, ok } = gitChurn(repo, { since: str(args.since) });
     return JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2);
   }
@@ -527,7 +680,7 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
   if (name === "search") {
     const query = str(args.query);
     if (!query) throw new Error("`query` is required");
-    const scan = scanRepo(repo, scanOpts);
+    const scan = getScan(repo, scanOpts);
     const limit = typeof args.limit === "number" ? args.limit : undefined;
     const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
     if (args.semantic === true) {
@@ -595,14 +748,28 @@ async function callTool(name: string, args: Record<string, unknown>): Promise<st
   }
   if (name === "check_rules") {
     const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload
-    const { graph } = buildIndexArtifacts(repo, scanOpts);
+    const { graph } = getArtifacts(repo, scanOpts);
     return JSON.stringify(checkRules(graph, rules), null, 2);
   }
   throw new Error(`unknown tool: ${name}`);
 }
 
-export async function runMcpServer(): Promise<void> {
-  await ensureGrammars(allGrammarKeys()); // AST tier when the sidecar is present
+export interface McpServerOptions {
+  // Override the serverInfo announced in the initialize response — for
+  // downstream consumers embedding this server under their own identity.
+  // Omitted fields keep the defaults (name "codeindex", ENGINE_VERSION).
+  serverInfo?: { name?: string; version?: string };
+}
+
+export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
+  const serverInfo = {
+    name: opts.serverInfo?.name ?? "codeindex",
+    version: opts.serverInfo?.version ?? ENGINE_VERSION,
+  };
+  // No startup warm: each scan-needing tool warms the present-language grammars
+  // for its repo before it runs (warmGrammarsForRepo re-derives them per call),
+  // so a session that never scans — or only touches one language — loads no
+  // unused wasm, and a language first seen mid-session still gets warmed.
 
   const send = (msg: Record<string, unknown>): void => {
     process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...msg }) + "\n");
@@ -635,7 +802,7 @@ export async function runMcpServer(): Promise<void> {
           result: {
             protocolVersion: "2024-11-05",
             capabilities: { tools: {} },
-            serverInfo: { name: "codeindex", version: ENGINE_VERSION },
+            serverInfo,
           },
         });
       } else if (req.method === "ping") {

package/src/pipeline.ts

@@ -1,13 +1,13 @@
 import type { Graph, SymbolIndex } from "./types.js";
 import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js";
-import { buildResolveContext } from "./resolve.js";
+import { resolveContextFor, symbolRefsFor } from "./derived.js";
 import { buildModules } from "./modules.js";
 import { buildGraph } from "./graph.js";
 import { detectCommunities } from "./community.js";
 import { applyCentrality } from "./centrality.js";
 import { computeTestMap } from "./tests-map.js";
 import { computeSurprises } from "./surprise.js";
-import { buildSymbolIndex, computeSymbolRefs } from "./render/symbols-json.js";
+import { buildSymbolIndex } from "./render/symbols-json.js";
 
 export interface BuildIndexOptions extends ScanOptions {
   // Stamped into the graph a consumer persists — lets it carry its own
@@ -29,8 +29,19 @@ export interface IndexArtifacts {
 // exact composition (and mutation order — it matters for byte-stable output)
 // that ultraindex's build performs before its prose rendering.
 export function buildIndexArtifacts(repo: string, opts: BuildIndexOptions = {}): IndexArtifacts {
-  const scan = scanRepo(repo, opts);
-  const ctx = buildResolveContext(scan);
+  return buildArtifactsFromScan(scanRepo(repo, opts), opts);
+}
+
+// Everything downstream of the scan: resolve → group → graph → communities →
+// centrality → tests-map → surprises → symbol index, in the same (load-bearing)
+// mutation order as buildIndexArtifacts — which is now just scanRepo + this.
+// Lets a caller that already holds a RepoScan build the artifacts without
+// re-walking the repo.
+export function buildArtifactsFromScan(scan: RepoScan, opts: BuildIndexOptions = {}): IndexArtifacts {
+  // Per-scan cache accessors (src/derived.ts): identical values to the direct
+  // builders, and building here PRE-WARMS the WeakMap so later queries on this
+  // same scan object (findReferences, findDeadCode, …) reuse the work.
+  const ctx = resolveContextFor(scan);
   const { modules, moduleOf } = buildModules(scan);
   const graph = buildGraph(scan, ctx, modules, moduleOf, opts.meta);
 
@@ -54,6 +65,6 @@ export function buildIndexArtifacts(repo: string, opts: BuildIndexOptions = {}):
   const surprises = computeSurprises(graph);
   if (surprises.length) graph.surprises = surprises;
 
-  const symbols = buildSymbolIndex(scan, computeSymbolRefs(scan));
+  const symbols = buildSymbolIndex(scan, symbolRefsFor(scan));
   return { scan, graph, symbols };
 }

package/src/query.ts

@@ -8,8 +8,8 @@ import { join } from "node:path";
 import type { CodeSymbol } from "./types.js";
 import type { RepoScan } from "./scan.js";
 import { readText } from "./walk.js";
-import { buildCallerIndex, type CallerSite } from "./callers.js";
-import { uniqueSymbolDefs } from "./graph.js";
+import type { CallerSite } from "./callers.js";
+import { callerIndexFor, uniqueDefsFor } from "./derived.js";
 import { byStr } from "./sort.js";
 
 const REFERENCE_KINDS = new Set(["reexport", "reexport-all", "default"]);
@@ -95,12 +95,15 @@ export function findReferences(scan: RepoScan, name: string): SymbolReferences {
   }
   defs.sort((a, b) => byStr(a.file, b.file) || a.line - b.line);
 
-  const index = buildCallerIndex(scan);
+  const index = callerIndexFor(scan);
   const entry = index.get(name);
-  const callSites = entry ? entry.callers : [];
+  // COPY, not alias: the caller index is memoized per scan (src/derived.ts),
+  // so handing out the cached array would let a consumer mutation poison
+  // every later findReferences on this scan.
+  const callSites = entry ? [...entry.callers] : [];
 
   const referencingFiles = new Set<string>();
-  const unique = uniqueSymbolDefs(scan);
+  const unique = uniqueDefsFor(scan);
   const defFile = unique.get(name);
   for (const f of scan.files) {
     if (f.rel === defFile) continue;

package/src/scan.ts

@@ -1,6 +1,6 @@
 import { basename } from "node:path";
 import type { FileRecord } from "./types.js";
-import { walk, readText } from "./walk.js";
+import { walk, readText, type WalkResult } from "./walk.js";
 import { headCommit } from "./git.js";
 import { sha1 } from "./hash.js";
 import { classify, MARKDOWN_EXT } from "./classify.js";
@@ -27,6 +27,20 @@ export interface RepoScan {
   // rules — see WalkResult.excluded). Surfaced so a consumer can report how
   // much of the tree was filtered out of the index.
   excluded: number;
+  // Change-tracking flags. DERIVED ONLY — they never influence records or
+  // ordering, so artifacts stay byte-identical whether or not anyone reads them.
+  //
+  // True iff a cache was supplied AND every kept file reused its cached record
+  // (via the stat fastpath or an exact content-hash match) AND the kept file
+  // set equals the cache's key set — i.e. this scan proved the indexed content
+  // identical to the previous build's. Docs never take the fastpath (see the
+  // docs exemption below), so their contribution is always an exact hash check.
+  contentUnchanged: boolean;
+  // True iff persisting this scan's cache would change at least one byte of the
+  // previous cache: some kept file's (hash, size, mtimeMs) differs from its
+  // cache entry, the kept file set differs from the cache's key set, or no
+  // cache was supplied at all. False means a cache rewrite would be pure churn.
+  cacheDirty: boolean;
 }
 
 export interface ScanOptions {
@@ -53,6 +67,12 @@ export interface ScanOptions {
   cache?: Map<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }>;
   // Disable the stat fastpath: read and re-hash every file (see BuildOptions).
   fullHash?: boolean;
+  // A walk result to use INSTEAD of walking here. MUST come from
+  // walk(root, <the same options as this scan>) — a walk of a different root
+  // or with different walk options would silently desynchronize the records
+  // from the tree. Exists for callers that already walked (e.g. a freshness
+  // probe) so scanRepo does not pay a second full directory traversal.
+  precomputedWalk?: WalkResult;
 }
 
 function countLines(s: string): number {
@@ -68,12 +88,14 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
   const scoped = opts.scope ? [...(opts.include ?? []), `${opts.scope.replace(/\/+$/, "")}/**`] : opts.include;
   const include = compileGlobs(scoped);
   const exclude = compileGlobs(opts.exclude);
-  const { files: walked, capped, excluded } = walk(root, {
-    maxFileBytes: opts.maxBytes,
-    maxFiles: opts.maxFiles,
-    gitignore: opts.gitignore,
-    ignoreDirs: opts.ignoreDirs,
-  });
+  const { files: walked, capped, excluded } =
+    opts.precomputedWalk ??
+    walk(root, {
+      maxFileBytes: opts.maxBytes,
+      maxFiles: opts.maxFiles,
+      gitignore: opts.gitignore,
+      ignoreDirs: opts.ignoreDirs,
+    });
   // Never index our own output (e.g. a committed `docs/ultraindex/`), or builds
   // would describe the encyclopedia instead of the code.
   const outPrefix = opts.out ? opts.out.replace(/\/+$/, "") + "/" : null;
@@ -83,6 +105,15 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
   const docText = new Map<string, string>();
   const mtimes = new Map<string, number>();
 
+  // Change-tracking accumulators (see RepoScan.contentUnchanged / cacheDirty).
+  // Derived only — nothing below feeds them back into records or ordering.
+  const cache = opts.cache;
+  // Every kept file so far reused its cached record (fastpath or hash match).
+  // Starts false with no cache: nothing can be proven unchanged against nothing.
+  let allReused = cache !== undefined;
+  // With no cache, persisting one is all new bytes — dirty by definition.
+  let cacheDirty = cache === undefined;
+
   for (const f of walked) {
     if (outPrefix && (f.abs === opts.out || f.abs.startsWith(outPrefix))) continue;
     if (include && !include(f.rel)) continue;
@@ -125,9 +156,17 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
     if (cached && cached.hash === hash) {
       files.push(cached.record);
       if (kind === "doc" && content) docText.set(f.rel, content);
+      // Content proven identical, but a (size, mtimeMs) drift — e.g. a bare
+      // touch, or an old cache without stat keys — still rewrites cache bytes.
+      if (cached.size !== f.size || cached.mtimeMs !== f.mtimeMs) cacheDirty = true;
       continue;
     }
 
+    // Past both reuse paths: this file is new to the cache or its content
+    // changed — the scan is not proven unchanged and the cache must be rewritten.
+    allReused = false;
+    cacheDirty = true;
+
     const record: FileRecord = {
       rel: f.rel,
       ext: f.ext,
@@ -176,5 +215,24 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
   }
 
   files.sort(byKey((f) => f.rel));
-  return { root, commit: headCommit(root), files, languages, docText, mtimes, capped, excluded };
+  // File-set equality: reuse requires a cache entry and rels are unique, so
+  // with allReused the kept set is a subset of the cache keys — equality is
+  // then exactly a count match. A count mismatch (file added, deleted, or
+  // newly filtered) also means the persisted key set changes: dirty.
+  if (cache !== undefined && files.length !== cache.size) {
+    allReused = false;
+    cacheDirty = true;
+  }
+  return {
+    root,
+    commit: headCommit(root),
+    files,
+    languages,
+    docText,
+    mtimes,
+    capped,
+    excluded,
+    contentUnchanged: allReused,
+    cacheDirty,
+  };
 }

package/src/types.ts

@@ -1,7 +1,7 @@
 // Single source of truth for the engine version the bundle reports. Kept in
 // lockstep with package.json by the release pipeline. Do not edit by hand
 // outside a release.
-export const ENGINE_VERSION = "2.13.0";
+export const ENGINE_VERSION = "2.14.0";
 
 // Bumped whenever the on-disk artifact shape changes, so a consumer can reject
 // an index written by an incompatible engine instead of misreading it. The

package/src/walk.ts

@@ -1,4 +1,4 @@
-import { readdirSync, statSync, lstatSync, readFileSync, realpathSync } from "node:fs";
+import { readdirSync, statSync, lstatSync, readFileSync, realpathSync, type Dirent } from "node:fs";
 import { join, relative, sep, extname } from "node:path";
 import { parseGitignore, isIgnored, type IgnoreRule } from "./ignore.js";
 
@@ -113,27 +113,44 @@ export function walk(root: string, opts: WalkOptions = {}): WalkResult {
     if (seenDirs.has(real)) continue;
     seenDirs.add(real);
     if (!contained(real)) continue; // dir symlink escaping the repo
-    let entries: string[];
+    let entries: Dirent[];
     try {
       // Sorted so the walk order — and therefore WHICH files survive a
-      // maxFiles cap — is identical across filesystems and machines.
-      entries = readdirSync(frame.dir).sort();
+      // maxFiles cap — is identical across filesystems and machines. Dirents
+      // sort by .name under the same code-unit comparison the bare-string
+      // sort used, so the order is byte-identical to the previous readdir.
+      entries = readdirSync(frame.dir, { withFileTypes: true }).sort((a, b) =>
+        a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
+      );
     } catch {
       continue;
     }
     let rules = frame.rules;
-    if (useGitignore && entries.includes(".gitignore")) {
+    if (useGitignore && entries.some((e) => e.name === ".gitignore")) {
       const parsed = parseGitignore(readText(join(frame.dir, ".gitignore")), frame.rel);
       if (parsed.length) rules = [...rules, ...parsed];
     }
-    for (const name of entries) {
+    for (const entry of entries) {
+      const name = entry.name;
       const abs = join(frame.dir, name);
       const rel = frame.rel ? `${frame.rel}/${name}` : name;
+      // The dirent type IS the lstat type (Node lstats internally when the
+      // filesystem can't supply it), so no lstat is needed to detect links.
+      const isLink = entry.isSymbolicLink();
+      // Ignored-directory boundary (node_modules, .git…): skip on the dirent
+      // type alone — ZERO stat syscalls. A symlink reports isDirectory()
+      // false on its dirent and falls through to the stat-based
+      // classification below, so a link named node_modules still classifies
+      // by its target exactly as before.
+      if (entry.isDirectory() && ignoreDirs.has(name)) continue;
       let st;
-      let isLink: boolean;
       try {
-        st = statSync(abs);
-        isLink = lstatSync(abs).isSymbolicLink();
+        // Non-links: a single lstatSync supplies isDirectory/isFile/size/
+        // mtimeMs — field-for-field what the previous statSync returned,
+        // since with no link to follow the two calls are identical. Links:
+        // keep the following statSync so the entry classifies by its TARGET;
+        // a broken link throws here and is skipped, same as before.
+        st = isLink ? statSync(abs) : lstatSync(abs);
       } catch {
         continue;
       }