@maxgfr/codeindex 2.13.0 → 2.15.0

package/src/engine-cli.ts

@@ -1,13 +1,21 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
 import { SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord } from "./types.js";
 import { ENGINE_VERSION } from "./types.js";
-import { ensureGrammars, allGrammarKeys } from "./ast/loader.js";
-import { buildIndexArtifacts, type BuildIndexOptions } from "./pipeline.js";
+import { ensureGrammars, grammarKeysForExts, resolveGrammarsTier, sharedGrammarsCacheDir } from "./ast/loader.js";
+import {
+  resolveGrammarsPullTarget,
+  fetchGrammarsTarball,
+  fetchExpectedSha256,
+  extractGrammarsTarball,
+} from "./ast/grammars-pull.js";
+import { buildIndexArtifacts, buildArtifactsFromScan, type BuildIndexOptions } from "./pipeline.js";
+import { sha1 } from "./hash.js";
 import { renderGraphJson } from "./render/graph-json.js";
 import { renderSymbolsJson } from "./render/symbols-json.js";
 import { renderScip } from "./render/scip.js";
 import { scanRepo } from "./scan.js";
+import { walk, type WalkResult } from "./walk.js";
 import { buildCallerIndex } from "./callers.js";
 import { detectWorkspaces } from "./workspaces.js";
 import { gitChurn } from "./git.js";
@@ -60,6 +68,14 @@ Commands:
                                the source with CODEINDEX_EMBED_URL
                 embed serve    Print (or --run) the docker command that starts the
                                containerized embedding server (rich tier)
+  grammars    Tree-sitter wasm grammars (optional AST tier; regex without them).
+              Precedence: bundle-adjacent > CODEINDEX_GRAMMARS_DIR > shared cache:
+                grammars status  Active tier (adjacent/env/cache/none), resolved
+                                 dir, pinned ENGINE_VERSION, pull-needed (JSON)
+                grammars pull    Fetch the per-release grammars-<version>.tar.gz
+                                 asset into the shared cache (sha256-verified,
+                                 atomic). Override the source with
+                                 CODEINDEX_GRAMMARS_URL
   rules       Architecture rules (forbidden edges, cycles, orphans) validated
               against the link-graph: --config <codeindex.rules.json>; exits 1
               on any error-severity violation (a CI gate)
@@ -175,7 +191,7 @@ function emit(content: string, out?: string): void {
   else process.stdout.write(content);
 }
 
-function scanOptions(flags: CliFlags): BuildIndexOptions {
+function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexOptions {
   return {
     include: flags.include.length ? flags.include : undefined,
     exclude: flags.exclude.length ? flags.exclude : undefined,
@@ -185,9 +201,22 @@ function scanOptions(flags: CliFlags): BuildIndexOptions {
     maxFiles: flags.maxFiles,
     maxBytes: flags.maxBytes,
     maxCallsPerFile: flags.maxCalls,
+    // The walk performed once in runCli to warm the present-language grammars,
+    // reused here so scanRepo does not traverse the tree a second time. Absent
+    // for --no-ast / scan-less commands: scanRepo walks itself, unchanged.
+    precomputedWalk,
   };
 }
 
+// Commands that never walk/scan the file tree — they read git/grep directly or
+// only manage the grammar cache, so they must warm NO tree-sitter grammar (the
+// CLI previously warmed every one unconditionally). `embed` is scan-only for its
+// `build` subcommand; the other embed subcommands (status/pull/serve) are
+// excluded by the positional check at the warm site. `grammars` (status/pull)
+// resolves/downloads the wasms itself and must not warm them.
+// version/help/mcp return before we get there.
+const SCANLESS_COMMANDS = new Set(["grep", "churn", "coupling", "workspaces", "grammars"]);
+
 export async function runCli(argv: string[]): Promise<void> {
   const [cmd, ...rest] = argv;
   if (!cmd || cmd === "help" || cmd === "--help" || cmd === "-h") {
@@ -206,7 +235,25 @@ export async function runCli(argv: string[]): Promise<void> {
 
   const flags = parseFlags(rest);
   if (!existsSync(flags.repo)) throw new Error(`--repo path does not exist: ${flags.repo}`);
-  if (!flags.noAst) await ensureGrammars(allGrammarKeys());
+
+  // Warm ONLY the grammars for languages actually present, and only for commands
+  // that scan the file tree. Scan-less commands (grep, churn, coupling,
+  // workspaces, embed status|pull|serve) load no grammar at all; version/help/mcp
+  // already returned above. The walk is done ONCE here to derive the present
+  // extensions, then handed to the scan via precomputedWalk so the tree is
+  // traversed a single time. --no-ast keeps the regex tier: no walk, no warm —
+  // scanRepo walks itself, exactly as before.
+  const scans = !SCANLESS_COMMANDS.has(cmd) && !(cmd === "embed" && flags.positional !== "build");
+  let precomputedWalk: WalkResult | undefined;
+  if (scans && !flags.noAst) {
+    precomputedWalk = walk(flags.repo, {
+      maxFileBytes: flags.maxBytes,
+      maxFiles: flags.maxFiles,
+      gitignore: flags.gitignore,
+      ignoreDirs: flags.ignoreDirs.length ? flags.ignoreDirs : undefined,
+    });
+    await ensureGrammars(grammarKeysForExts(precomputedWalk.files.map((f) => f.ext)));
+  }
 
   if (cmd === "index") {
     if (!flags.out) throw new Error("index needs --out <dir>");
@@ -215,47 +262,143 @@ export async function runCli(argv: string[]): Promise<void> {
     // Incremental cache: reuse per-file records when (schema, extractor) match —
     // same invalidation discipline as ultraindex's cache.json.
     const cachePath = join(outDir, "cache.json");
-    let cache: Map<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }> | undefined;
+    type CacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number };
+    // ADDITIVE meta keys describing the artifacts the cache-writing run put on
+    // disk. Old engines ignore them (they only check schema/extractor above);
+    // old caches lacking them simply never take the fastpath below (their
+    // per-file records are still reused). cache.json embeds mtimes, so it was
+    // never cross-machine byte-reproducible — no determinism surface changes.
+    type CacheMeta = {
+      engineVersion?: string;
+      commit?: string;
+      graphSha1?: string;
+      symbolsSha1?: string;
+      embed?: { embedVersion?: number; modelId?: string; sha1?: string };
+    };
+    let cache: Map<string, CacheEntry> | undefined;
+    let meta: CacheMeta = {};
     try {
       const parsed = JSON.parse(readFileSync(cachePath, "utf8")) as {
         schemaVersion: number;
         extractorVersion: number;
-        files: Record<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }>;
-      };
+        files: Record<string, CacheEntry>;
+      } & CacheMeta;
       if (parsed.schemaVersion === SCHEMA_VERSION && parsed.extractorVersion === EXTRACTOR_VERSION) {
         cache = new Map(Object.entries(parsed.files));
+        meta = {
+          engineVersion: parsed.engineVersion,
+          commit: parsed.commit,
+          graphSha1: parsed.graphSha1,
+          symbolsSha1: parsed.symbolsSha1,
+          embed: parsed.embed,
+        };
       }
     } catch {
       // no cache yet (or unreadable) — cold build
     }
-    const { scan, graph, symbols } = buildIndexArtifacts(flags.repo, { ...scanOptions(flags), cache, out: outDir });
-    writeFileSync(join(outDir, "graph.json"), renderGraphJson(graph));
-    writeFileSync(join(outDir, "symbols.json"), renderSymbolsJson(symbols));
-    const files: Record<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }> = {};
-    for (const f of scan.files) {
-      const entry: { hash: string; record: FileRecord; size?: number; mtimeMs?: number } = { hash: f.hash, record: f, size: f.size };
-      const mtime = scan.mtimes.get(f.rel);
-      if (mtime !== undefined) entry.mtimeMs = mtime;
-      files[f.rel] = entry;
-    }
-    writeFileSync(
-      cachePath,
-      JSON.stringify({ schemaVersion: SCHEMA_VERSION, extractorVersion: EXTRACTOR_VERSION, files }) + "\n",
-    );
-    // Deterministic embeddings sidecar: written next to graph.json ONLY when a
-    // model asset is present (opt-in). Silently skipped otherwise — no model, no
-    // embeddings.bin, no impact on the graph/symbols consumers.
-    let embedNote = "";
+    const scan = scanRepo(flags.repo, { ...scanOptions(flags, precomputedWalk), cache, out: outDir });
     const modelDir = resolveEmbedModelDir(flags.repo);
     const model = modelDir ? loadEmbedModel(modelDir) : undefined;
-    if (model) {
-      const index = buildEmbeddingIndex(scan, model);
-      writeFileSync(join(outDir, "embeddings.bin"), serializeEmbeddings(index));
-      embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+
+    const graphPath = join(outDir, "graph.json");
+    const symbolsPath = join(outDir, "symbols.json");
+    const embedPath = join(outDir, "embeddings.bin");
+    // sha of an on-disk artifact, or undefined when it is missing/unreadable —
+    // never equal to a defined meta sha, so a deleted artifact fails the guard.
+    const artifactSha = (path: string): string | undefined => {
+      try {
+        return sha1(readFileSync(path));
+      } catch {
+        return undefined;
+      }
+    };
+    const writeCache = (out: Pick<CacheMeta, "graphSha1" | "symbolsSha1" | "embed">): void => {
+      const files: Record<string, CacheEntry> = {};
+      for (const f of scan.files) {
+        const entry: CacheEntry = { hash: f.hash, record: f, size: f.size };
+        const mtime = scan.mtimes.get(f.rel);
+        if (mtime !== undefined) entry.mtimeMs = mtime;
+        files[f.rel] = entry;
+      }
+      // Fixed key order; JSON.stringify drops the undefined-valued keys
+      // (commit outside a git worktree, embed without a model) cleanly.
+      writeFileSync(
+        cachePath,
+        JSON.stringify({
+          schemaVersion: SCHEMA_VERSION,
+          extractorVersion: EXTRACTOR_VERSION,
+          engineVersion: ENGINE_VERSION,
+          commit: scan.commit,
+          graphSha1: out.graphSha1,
+          symbolsSha1: out.symbolsSha1,
+          embed: out.embed,
+          files,
+        }) + "\n",
+      );
+    };
+
+    // FASTPATH GUARD — skip the whole downstream pipeline only when this scan
+    // is proven identical to the run that wrote the on-disk artifacts:
+    // contentUnchanged means this scan's records are object-identical to that
+    // run's; downstream is a pure function of (records, docText, commit,
+    // meta-opts) and the CLI never sets meta/previousCommunities;
+    // engineVersion pins the version stamp; commit must match because
+    // graph.json embeds it (identical trees under a new HEAD must rebuild);
+    // the shas prove the on-disk bytes are that run's output. ANY failure —
+    // deleted or tampered artifacts included — falls through to the full
+    // build, which rewrites everything (self-healing).
+    const embedUnchanged =
+      !model ||
+      (meta.embed !== undefined &&
+        meta.embed.embedVersion === EMBED_VERSION &&
+        meta.embed.modelId === model.modelId &&
+        meta.embed.sha1 !== undefined &&
+        artifactSha(embedPath) === meta.embed.sha1);
+    const fastpath =
+      scan.contentUnchanged &&
+      meta.engineVersion === ENGINE_VERSION &&
+      meta.commit === scan.commit &&
+      meta.graphSha1 !== undefined &&
+      artifactSha(graphPath) === meta.graphSha1 &&
+      meta.symbolsSha1 !== undefined &&
+      artifactSha(symbolsPath) === meta.symbolsSha1 &&
+      embedUnchanged;
+
+    if (fastpath) {
+      // Artifacts verified byte-identical to what this build would produce —
+      // leave them untouched. Rewrite cache.json only when the scan says its
+      // bytes would change (e.g. an mtime drifted); the meta is carried
+      // forward verbatim since the guard just proved it describes the disk.
+      if (scan.cacheDirty) writeCache(meta);
+      process.stderr.write(
+        `codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${scan.capped ? " (capped)" : ""} (unchanged — artifacts reused)\n`,
+      );
+    } else {
+      const { graph, symbols } = buildArtifactsFromScan(scan);
+      const graphJson = renderGraphJson(graph);
+      const symbolsJson = renderSymbolsJson(symbols);
+      writeFileSync(graphPath, graphJson);
+      writeFileSync(symbolsPath, symbolsJson);
+      // Deterministic embeddings sidecar: written next to graph.json ONLY when a
+      // model asset is present (opt-in). Silently skipped otherwise — no model, no
+      // embeddings.bin, no impact on the graph/symbols consumers.
+      let embedNote = "";
+      let embedMeta: CacheMeta["embed"];
+      if (model) {
+        const index = buildEmbeddingIndex(scan, model);
+        const bytes = serializeEmbeddings(index);
+        writeFileSync(embedPath, bytes);
+        embedMeta = { embedVersion: EMBED_VERSION, modelId: model.modelId, sha1: sha1(bytes) };
+        embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+      }
+      // cache.json is written LAST so its meta always describes artifacts that
+      // are already on disk — a crash mid-way leaves stale meta whose shas
+      // fail the guard on the next run (safe: it just rebuilds).
+      writeCache({ graphSha1: sha1(graphJson), symbolsSha1: sha1(symbolsJson), embed: embedMeta });
+      process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
     }
-    process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
   } else if (cmd === "scan") {
-    const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     const summary = {
       engineVersion: ENGINE_VERSION,
       commit: scan.commit,
@@ -265,13 +408,13 @@ export async function runCli(argv: string[]): Promise<void> {
     };
     emit(JSON.stringify(summary, null, 2) + "\n", flags.out);
   } else if (cmd === "graph") {
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderGraphJson(graph), flags.out);
   } else if (cmd === "symbols") {
-    const { symbols } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { symbols } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderSymbolsJson(symbols), flags.out);
   } else if (cmd === "scip") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const bytes = renderScip(scan, { projectRoot: flags.projectRoot });
     const out = flags.out ?? resolve("index.scip");
     if (out === "-") process.stdout.write(Buffer.from(bytes));
@@ -280,14 +423,14 @@ export async function runCli(argv: string[]): Promise<void> {
       process.stderr.write(`codeindex: SCIP index → ${out} (${bytes.length} bytes)\n`);
     }
   } else if (cmd === "callers") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const index = buildCallerIndex(scan, undefined, { recall: flags.recall });
     const obj: Record<string, unknown> = {};
     for (const [name, entry] of index) obj[name] = entry;
     emit(JSON.stringify(obj, null, 2) + "\n", flags.out);
   } else if (cmd === "search") {
     if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     if (flags.semantic) {
       const endpoint = resolveEmbedEndpoint();
       const lexical = (): void => {
@@ -392,7 +535,7 @@ export async function runCli(argv: string[]): Promise<void> {
       }
       const model = loadEmbedModel(modelDir)!;
       mkdirSync(flags.out, { recursive: true });
-      const scan = scanRepo(flags.repo, scanOptions(flags));
+      const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
       const index = buildEmbeddingIndex(scan, model);
       writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index));
       process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`);
@@ -429,10 +572,109 @@ export async function runCli(argv: string[]): Promise<void> {
     } else {
       throw new Error("embed needs a subcommand: status | build | pull | serve");
     }
+  } else if (cmd === "grammars") {
+    const sub = flags.positional;
+    const cacheDir = sharedGrammarsCacheDir();
+    if (sub === "status") {
+      // Report which tier furnishes the wasms (adjacent/env/cache/none), the
+      // resolved dir, the pinned ENGINE_VERSION the cache is keyed on, and
+      // whether a pull is needed (no runtime wasm resolvable → AST off, regex).
+      const info = resolveGrammarsTier();
+      const runtimePresent = info.dir ? existsSync(join(info.dir, "web-tree-sitter.wasm")) : false;
+      const target = resolveGrammarsPullTarget();
+      const status: Record<string, unknown> = {
+        engineVersion: ENGINE_VERSION,
+        tier: info.tier,
+        dir: info.dir ?? null,
+        cacheDir,
+        runtimePresent,
+        pullNeeded: !runtimePresent,
+        url: target.url,
+      };
+      emit(JSON.stringify(status, null, 2) + "\n", flags.out);
+    } else if (sub === "pull") {
+      // Default: the official per-release asset + its `.sha256` sidecar. A
+      // user-set CODEINDEX_GRAMMARS_URL overrides both (mirror/custom, no
+      // verification). Fetch the expected digest from the sidecar first; a
+      // missing sidecar degrades to an unverified pull with a note (never fatal).
+      const target = resolveGrammarsPullTarget();
+      let expected: string | undefined;
+      if (target.sha256Url) {
+        try {
+          expected = await fetchExpectedSha256(target.sha256Url);
+        } catch (e) {
+          process.stderr.write(
+            `codeindex: could not fetch checksum (${e instanceof Error ? e.message : String(e)}) — proceeding unverified\n`,
+          );
+        }
+      }
+      // Idempotent: the marker sibling records the digest of the tarball that
+      // populated cacheDir. If the runtime wasm is present AND the marker matches
+      // the freshly-fetched digest, the cache is already up to date — skip the
+      // ~22 MB download entirely. (Keeps cacheDir itself byte-identical to the
+      // tarball; the marker lives next to it, never inside it.)
+      const runtime = join(cacheDir, "web-tree-sitter.wasm");
+      const markerPath = join(dirname(cacheDir), `${ENGINE_VERSION}.sha256`);
+      if (existsSync(runtime) && expected && existsSync(markerPath)) {
+        let marker = "";
+        try {
+          marker = readFileSync(markerPath, "utf8").trim();
+        } catch {
+          // unreadable marker → fall through and re-pull
+        }
+        if (marker === expected) {
+          process.stderr.write(`codeindex: grammars already present at ${cacheDir} (up to date)\n`);
+          return;
+        }
+      }
+      process.stderr.write(`codeindex: fetching grammars from ${target.url} → ${cacheDir}\n`);
+      let bytes: Uint8Array;
+      try {
+        // Follows redirects (GitHub → CDN) and verifies sha256 when known.
+        bytes = await fetchGrammarsTarball(target.url, expected);
+      } catch (e) {
+        process.stderr.write(`codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`);
+        process.exitCode = 1;
+        return;
+      }
+      // Atomic install: extract into a tmp dir SIBLING to cacheDir (same
+      // filesystem → rename is atomic), sanity-check the runtime wasm landed,
+      // then swap it into place. A failure at any step discards the tmp dir and
+      // leaves any existing cache untouched — a half-populated cache never shows.
+      let tmp: string | undefined;
+      try {
+        mkdirSync(dirname(cacheDir), { recursive: true });
+        tmp = mkdtempSync(join(dirname(cacheDir), ".grammars-tmp-"));
+        extractGrammarsTarball(bytes, tmp);
+        if (!existsSync(join(tmp, "web-tree-sitter.wasm"))) {
+          throw new Error("archive is missing web-tree-sitter.wasm");
+        }
+        if (existsSync(cacheDir)) rmSync(cacheDir, { recursive: true, force: true });
+        renameSync(tmp, cacheDir);
+        tmp = undefined;
+        if (expected) writeFileSync(markerPath, expected + "\n");
+      } catch (e) {
+        if (tmp) {
+          try {
+            rmSync(tmp, { recursive: true, force: true });
+          } catch {
+            // best-effort cleanup
+          }
+        }
+        process.stderr.write(
+          `codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`,
+        );
+        process.exitCode = 1;
+        return;
+      }
+      process.stderr.write(`codeindex: grammars extracted → ${cacheDir}\n`);
+    } else {
+      throw new Error("grammars needs a subcommand: status | pull");
+    }
   } else if (cmd === "rules") {
     if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
     const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8")));
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     const violations = checkRules(graph, rules);
     const errors = violations.filter((v) => v.severity === "error").length;
     emit(JSON.stringify({ errors, warnings: violations.length - errors, violations }, null, 2) + "\n", flags.out);
@@ -453,26 +695,26 @@ export async function runCli(argv: string[]): Promise<void> {
     for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!;
     emit(JSON.stringify({ ok, churn: sorted }, null, 2) + "\n", flags.out);
   } else if (cmd === "repomap") {
-    const { scan, graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { scan, graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderRepoMap(scan, graph, { budgetTokens: flags.budgetTokens }), flags.out);
   } else if (cmd === "hotspots") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const { churn, ok } = gitChurn(flags.repo, { since: flags.since });
     emit(JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2) + "\n", flags.out);
   } else if (cmd === "coupling") {
     const { ok, couplings } = changeCoupling(flags.repo, { since: flags.since });
     emit(JSON.stringify({ ok, couplings }, null, 2) + "\n", flags.out);
   } else if (cmd === "deadcode") {
-    emit(JSON.stringify(findDeadCode(scanRepo(flags.repo, scanOptions(flags))), null, 2) + "\n", flags.out);
+    emit(JSON.stringify(findDeadCode(scanRepo(flags.repo, scanOptions(flags, precomputedWalk))), null, 2) + "\n", flags.out);
   } else if (cmd === "complexity") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     emit(JSON.stringify(symbolComplexity(scan, flags.positional), null, 2) + "\n", flags.out);
   } else if (cmd === "risk") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const { churn, ok } = gitChurn(flags.repo, { since: flags.since });
     emit(JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2) + "\n", flags.out);
   } else if (cmd === "mermaid") {
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderMermaid(graph, { module: flags.positional }), flags.out);
   } else if (cmd === "grep") {
     if (!flags.positional) throw new Error("grep needs a pattern: cli.mjs grep <pattern> --repo <dir>");

package/src/engine.ts

@@ -45,8 +45,23 @@ export { extractMarkdown } from "./extract/markdown.js";
 export type { MarkdownInfo } from "./extract/markdown.js";
 
 // AST tier (optional — a no-op without the grammar wasm sidecar).
-export { ensureGrammars, allGrammarKeys, grammarKeyForExt, grammarReady } from "./ast/loader.js";
+export { ensureGrammars, allGrammarKeys, grammarKeysForExts, grammarKeyForExt, grammarReady } from "./ast/loader.js";
 export { extractAst } from "./ast/extract.js";
+// Grammars resolution + the slim pull/cache tier (v2.14.0): resolve the wasm
+// dir across bundle-adjacent → env → shared cache → none, and fetch the
+// committed wasms into the shared cache when none ship next to the bundle
+// (`codeindex grammars pull|status`). Additive; opt-in; offline-safe.
+export { resolveGrammarsDir, resolveGrammarsTier, sharedGrammarsCacheDir } from "./ast/loader.js";
+export type { GrammarsTier, GrammarsTierName } from "./ast/loader.js";
+export {
+  DEFAULT_GRAMMARS_URL,
+  resolveGrammarsPullTarget,
+  fetchGrammarsTarball,
+  fetchExpectedSha256,
+  extractTarInto,
+  extractGrammarsTarball,
+} from "./ast/grammars-pull.js";
+export type { GrammarsPullTarget } from "./ast/grammars-pull.js";
 
 // Resolution + modules + graph tier.
 export { buildResolveContext, resolveImport, resolveDocLink } from "./resolve.js";
@@ -78,8 +93,9 @@ export { renderGraphJson } from "./render/graph-json.js";
 export { renderScip } from "./render/scip.js";
 export type { RenderScipOptions } from "./render/scip.js";
 
-// One-call pipeline.
-export { buildIndexArtifacts } from "./pipeline.js";
+// One-call pipeline (buildArtifactsFromScan: the scan-onward half, for callers
+// that already hold a RepoScan).
+export { buildIndexArtifacts, buildArtifactsFromScan } from "./pipeline.js";
 export type { BuildIndexOptions, IndexArtifacts } from "./pipeline.js";
 
 // Git utilities.
@@ -161,6 +177,7 @@ export type { RepoMapOptions } from "./repomap.js";
 
 // MCP server over stdio (also reachable as `engine.mjs mcp`).
 export { runMcpServer } from "./mcp.js";
+export type { McpServerOptions } from "./mcp.js";
 
 // General-purpose helpers shared by consumers (deterministic, dependency-free).
 export { sha1, shortHash } from "./hash.js";

package/src/hash.ts

@@ -2,8 +2,11 @@ import { createHash } from "node:crypto";
 
 // Stable content hash used for the manifest's staleness oracle and for the
 // `ui:gen` region fingerprints. sha1 is plenty for change detection (not
-// security) and keeps the manifest compact.
-export function sha1(s: string): string {
+// security) and keeps the manifest compact. Accepts raw bytes too (additive)
+// so binary artifacts (e.g. embeddings.bin) hash without a lossy decode; a
+// string is hashed as its UTF-8 bytes — identical to hashing the bytes
+// writeFileSync would put on disk for that string.
+export function sha1(s: string | Uint8Array): string {
   return createHash("sha1").update(s).digest("hex");
 }