@maxgfr/codeindex 2.12.0 → 2.14.0

package/src/engine-cli.ts

@@ -1,13 +1,21 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
 import { SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord } from "./types.js";
 import { ENGINE_VERSION } from "./types.js";
-import { ensureGrammars, allGrammarKeys } from "./ast/loader.js";
-import { buildIndexArtifacts, type BuildIndexOptions } from "./pipeline.js";
+import { ensureGrammars, grammarKeysForExts, resolveGrammarsTier, sharedGrammarsCacheDir } from "./ast/loader.js";
+import {
+  resolveGrammarsPullTarget,
+  fetchGrammarsTarball,
+  fetchExpectedSha256,
+  extractGrammarsTarball,
+} from "./ast/grammars-pull.js";
+import { buildIndexArtifacts, buildArtifactsFromScan, type BuildIndexOptions } from "./pipeline.js";
+import { sha1 } from "./hash.js";
 import { renderGraphJson } from "./render/graph-json.js";
 import { renderSymbolsJson } from "./render/symbols-json.js";
 import { renderScip } from "./render/scip.js";
 import { scanRepo } from "./scan.js";
+import { walk, type WalkResult } from "./walk.js";
 import { buildCallerIndex } from "./callers.js";
 import { detectWorkspaces } from "./workspaces.js";
 import { gitChurn } from "./git.js";
@@ -19,7 +27,7 @@ import { symbolComplexity, riskHotspots } from "./complexity.js";
 import { renderMermaid } from "./viz.js";
 import { searchIndex } from "./bm25.js";
 import { checkRules, parseRules } from "./rules.js";
-import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, resolveEmbedPullUrl, fetchEmbedModel } from "./embed/model.js";
+import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, parseEmbedModel, resolveEmbedPullUrl, fetchEmbedModel } from "./embed/model.js";
 import { buildEmbeddingIndex, serializeEmbeddings } from "./embed/index.js";
 import { searchSemantic } from "./embed/search.js";
 import {
@@ -60,6 +68,14 @@ Commands:
                                the source with CODEINDEX_EMBED_URL
                 embed serve    Print (or --run) the docker command that starts the
                                containerized embedding server (rich tier)
+  grammars    Tree-sitter wasm grammars (optional AST tier; regex without them).
+              Precedence: bundle-adjacent > CODEINDEX_GRAMMARS_DIR > shared cache:
+                grammars status  Active tier (adjacent/env/cache/none), resolved
+                                 dir, pinned ENGINE_VERSION, pull-needed (JSON)
+                grammars pull    Fetch the per-release grammars-<version>.tar.gz
+                                 asset into the shared cache (sha256-verified,
+                                 atomic). Override the source with
+                                 CODEINDEX_GRAMMARS_URL
   rules       Architecture rules (forbidden edges, cycles, orphans) validated
               against the link-graph: --config <codeindex.rules.json>; exits 1
               on any error-severity violation (a CI gate)
@@ -80,8 +96,11 @@ Flags:
   --exclude <glob>    Exclude matching paths (repeatable)
   --scope <dir>       Restrict to one directory (sugar for --include '<dir>/**')
   --no-gitignore      Do not honor .gitignore files (default: honored)
+  --ignore-dir <name> Directory names to skip (repeatable) — REPLACES the
+                      default ignored-directory set, never merges with it
   --max-files <n>     Cap walked files (default 20000)
   --max-bytes <n>     Skip files above this size (default 1 MiB)
+  --max-calls <n>     Per-file call-site cap for extraction (default 512)
   --no-ast            Skip tree-sitter grammars even when present (regex tier)
   --config <file>     Rules config for \`rules\` (JSON: [{name, from, to, …}])
   --limit <n>         Max results for \`search\` (default 20)
@@ -103,8 +122,10 @@ interface CliFlags {
   exclude: string[];
   scope?: string;
   gitignore: boolean;
+  ignoreDirs: string[];
   maxFiles?: number;
   maxBytes?: number;
+  maxCalls?: number;
   noAst: boolean;
   since?: string;
   ignoreCase?: boolean;
@@ -121,7 +142,7 @@ interface CliFlags {
 }
 
 function parseFlags(args: string[]): CliFlags {
-  const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true, semantic: false };
+  const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, ignoreDirs: [], noAst: false, fuzzy: true, semantic: false };
   for (let i = 0; i < args.length; i++) {
     const a = args[i]!;
     const next = (): string => {
@@ -144,8 +165,10 @@ function parseFlags(args: string[]): CliFlags {
     else if (a === "--exclude") flags.exclude.push(next());
     else if (a === "--scope") flags.scope = next();
     else if (a === "--no-gitignore") flags.gitignore = false;
+    else if (a === "--ignore-dir") flags.ignoreDirs.push(next());
     else if (a === "--max-files") flags.maxFiles = num();
     else if (a === "--max-bytes") flags.maxBytes = num();
+    else if (a === "--max-calls") flags.maxCalls = num();
     else if (a === "--ignore-case") flags.ignoreCase = true;
     else if (a === "--max-hits") flags.maxHits = num();
     else if (a === "--budget-tokens") flags.budgetTokens = num();
@@ -168,17 +191,32 @@ function emit(content: string, out?: string): void {
   else process.stdout.write(content);
 }
 
-function scanOptions(flags: CliFlags): BuildIndexOptions {
+function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexOptions {
   return {
     include: flags.include.length ? flags.include : undefined,
     exclude: flags.exclude.length ? flags.exclude : undefined,
     scope: flags.scope,
     gitignore: flags.gitignore,
+    ignoreDirs: flags.ignoreDirs.length ? flags.ignoreDirs : undefined,
     maxFiles: flags.maxFiles,
     maxBytes: flags.maxBytes,
+    maxCallsPerFile: flags.maxCalls,
+    // The walk performed once in runCli to warm the present-language grammars,
+    // reused here so scanRepo does not traverse the tree a second time. Absent
+    // for --no-ast / scan-less commands: scanRepo walks itself, unchanged.
+    precomputedWalk,
   };
 }
 
+// Commands that never walk/scan the file tree — they read git/grep directly or
+// only manage the grammar cache, so they must warm NO tree-sitter grammar (the
+// CLI previously warmed every one unconditionally). `embed` is scan-only for its
+// `build` subcommand; the other embed subcommands (status/pull/serve) are
+// excluded by the positional check at the warm site. `grammars` (status/pull)
+// resolves/downloads the wasms itself and must not warm them.
+// version/help/mcp return before we get there.
+const SCANLESS_COMMANDS = new Set(["grep", "churn", "coupling", "workspaces", "grammars"]);
+
 export async function runCli(argv: string[]): Promise<void> {
   const [cmd, ...rest] = argv;
   if (!cmd || cmd === "help" || cmd === "--help" || cmd === "-h") {
@@ -197,7 +235,25 @@ export async function runCli(argv: string[]): Promise<void> {
 
   const flags = parseFlags(rest);
   if (!existsSync(flags.repo)) throw new Error(`--repo path does not exist: ${flags.repo}`);
-  if (!flags.noAst) await ensureGrammars(allGrammarKeys());
+
+  // Warm ONLY the grammars for languages actually present, and only for commands
+  // that scan the file tree. Scan-less commands (grep, churn, coupling,
+  // workspaces, embed status|pull|serve) load no grammar at all; version/help/mcp
+  // already returned above. The walk is done ONCE here to derive the present
+  // extensions, then handed to the scan via precomputedWalk so the tree is
+  // traversed a single time. --no-ast keeps the regex tier: no walk, no warm —
+  // scanRepo walks itself, exactly as before.
+  const scans = !SCANLESS_COMMANDS.has(cmd) && !(cmd === "embed" && flags.positional !== "build");
+  let precomputedWalk: WalkResult | undefined;
+  if (scans && !flags.noAst) {
+    precomputedWalk = walk(flags.repo, {
+      maxFileBytes: flags.maxBytes,
+      maxFiles: flags.maxFiles,
+      gitignore: flags.gitignore,
+      ignoreDirs: flags.ignoreDirs.length ? flags.ignoreDirs : undefined,
+    });
+    await ensureGrammars(grammarKeysForExts(precomputedWalk.files.map((f) => f.ext)));
+  }
 
   if (cmd === "index") {
     if (!flags.out) throw new Error("index needs --out <dir>");
@@ -206,47 +262,143 @@ export async function runCli(argv: string[]): Promise<void> {
     // Incremental cache: reuse per-file records when (schema, extractor) match —
     // same invalidation discipline as ultraindex's cache.json.
     const cachePath = join(outDir, "cache.json");
-    let cache: Map<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }> | undefined;
+    type CacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number };
+    // ADDITIVE meta keys describing the artifacts the cache-writing run put on
+    // disk. Old engines ignore them (they only check schema/extractor above);
+    // old caches lacking them simply never take the fastpath below (their
+    // per-file records are still reused). cache.json embeds mtimes, so it was
+    // never cross-machine byte-reproducible — no determinism surface changes.
+    type CacheMeta = {
+      engineVersion?: string;
+      commit?: string;
+      graphSha1?: string;
+      symbolsSha1?: string;
+      embed?: { embedVersion?: number; modelId?: string; sha1?: string };
+    };
+    let cache: Map<string, CacheEntry> | undefined;
+    let meta: CacheMeta = {};
     try {
       const parsed = JSON.parse(readFileSync(cachePath, "utf8")) as {
         schemaVersion: number;
         extractorVersion: number;
-        files: Record<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }>;
-      };
+        files: Record<string, CacheEntry>;
+      } & CacheMeta;
       if (parsed.schemaVersion === SCHEMA_VERSION && parsed.extractorVersion === EXTRACTOR_VERSION) {
         cache = new Map(Object.entries(parsed.files));
+        meta = {
+          engineVersion: parsed.engineVersion,
+          commit: parsed.commit,
+          graphSha1: parsed.graphSha1,
+          symbolsSha1: parsed.symbolsSha1,
+          embed: parsed.embed,
+        };
       }
     } catch {
       // no cache yet (or unreadable) — cold build
     }
-    const { scan, graph, symbols } = buildIndexArtifacts(flags.repo, { ...scanOptions(flags), cache, out: outDir });
-    writeFileSync(join(outDir, "graph.json"), renderGraphJson(graph));
-    writeFileSync(join(outDir, "symbols.json"), renderSymbolsJson(symbols));
-    const files: Record<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }> = {};
-    for (const f of scan.files) {
-      const entry: { hash: string; record: FileRecord; size?: number; mtimeMs?: number } = { hash: f.hash, record: f, size: f.size };
-      const mtime = scan.mtimes.get(f.rel);
-      if (mtime !== undefined) entry.mtimeMs = mtime;
-      files[f.rel] = entry;
-    }
-    writeFileSync(
-      cachePath,
-      JSON.stringify({ schemaVersion: SCHEMA_VERSION, extractorVersion: EXTRACTOR_VERSION, files }) + "\n",
-    );
-    // Deterministic embeddings sidecar: written next to graph.json ONLY when a
-    // model asset is present (opt-in). Silently skipped otherwise — no model, no
-    // embeddings.bin, no impact on the graph/symbols consumers.
-    let embedNote = "";
+    const scan = scanRepo(flags.repo, { ...scanOptions(flags, precomputedWalk), cache, out: outDir });
     const modelDir = resolveEmbedModelDir(flags.repo);
     const model = modelDir ? loadEmbedModel(modelDir) : undefined;
-    if (model) {
-      const index = buildEmbeddingIndex(scan, model);
-      writeFileSync(join(outDir, "embeddings.bin"), serializeEmbeddings(index));
-      embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+
+    const graphPath = join(outDir, "graph.json");
+    const symbolsPath = join(outDir, "symbols.json");
+    const embedPath = join(outDir, "embeddings.bin");
+    // sha of an on-disk artifact, or undefined when it is missing/unreadable —
+    // never equal to a defined meta sha, so a deleted artifact fails the guard.
+    const artifactSha = (path: string): string | undefined => {
+      try {
+        return sha1(readFileSync(path));
+      } catch {
+        return undefined;
+      }
+    };
+    const writeCache = (out: Pick<CacheMeta, "graphSha1" | "symbolsSha1" | "embed">): void => {
+      const files: Record<string, CacheEntry> = {};
+      for (const f of scan.files) {
+        const entry: CacheEntry = { hash: f.hash, record: f, size: f.size };
+        const mtime = scan.mtimes.get(f.rel);
+        if (mtime !== undefined) entry.mtimeMs = mtime;
+        files[f.rel] = entry;
+      }
+      // Fixed key order; JSON.stringify drops the undefined-valued keys
+      // (commit outside a git worktree, embed without a model) cleanly.
+      writeFileSync(
+        cachePath,
+        JSON.stringify({
+          schemaVersion: SCHEMA_VERSION,
+          extractorVersion: EXTRACTOR_VERSION,
+          engineVersion: ENGINE_VERSION,
+          commit: scan.commit,
+          graphSha1: out.graphSha1,
+          symbolsSha1: out.symbolsSha1,
+          embed: out.embed,
+          files,
+        }) + "\n",
+      );
+    };
+
+    // FASTPATH GUARD — skip the whole downstream pipeline only when this scan
+    // is proven identical to the run that wrote the on-disk artifacts:
+    // contentUnchanged means this scan's records are object-identical to that
+    // run's; downstream is a pure function of (records, docText, commit,
+    // meta-opts) and the CLI never sets meta/previousCommunities;
+    // engineVersion pins the version stamp; commit must match because
+    // graph.json embeds it (identical trees under a new HEAD must rebuild);
+    // the shas prove the on-disk bytes are that run's output. ANY failure —
+    // deleted or tampered artifacts included — falls through to the full
+    // build, which rewrites everything (self-healing).
+    const embedUnchanged =
+      !model ||
+      (meta.embed !== undefined &&
+        meta.embed.embedVersion === EMBED_VERSION &&
+        meta.embed.modelId === model.modelId &&
+        meta.embed.sha1 !== undefined &&
+        artifactSha(embedPath) === meta.embed.sha1);
+    const fastpath =
+      scan.contentUnchanged &&
+      meta.engineVersion === ENGINE_VERSION &&
+      meta.commit === scan.commit &&
+      meta.graphSha1 !== undefined &&
+      artifactSha(graphPath) === meta.graphSha1 &&
+      meta.symbolsSha1 !== undefined &&
+      artifactSha(symbolsPath) === meta.symbolsSha1 &&
+      embedUnchanged;
+
+    if (fastpath) {
+      // Artifacts verified byte-identical to what this build would produce —
+      // leave them untouched. Rewrite cache.json only when the scan says its
+      // bytes would change (e.g. an mtime drifted); the meta is carried
+      // forward verbatim since the guard just proved it describes the disk.
+      if (scan.cacheDirty) writeCache(meta);
+      process.stderr.write(
+        `codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${scan.capped ? " (capped)" : ""} (unchanged — artifacts reused)\n`,
+      );
+    } else {
+      const { graph, symbols } = buildArtifactsFromScan(scan);
+      const graphJson = renderGraphJson(graph);
+      const symbolsJson = renderSymbolsJson(symbols);
+      writeFileSync(graphPath, graphJson);
+      writeFileSync(symbolsPath, symbolsJson);
+      // Deterministic embeddings sidecar: written next to graph.json ONLY when a
+      // model asset is present (opt-in). Silently skipped otherwise — no model, no
+      // embeddings.bin, no impact on the graph/symbols consumers.
+      let embedNote = "";
+      let embedMeta: CacheMeta["embed"];
+      if (model) {
+        const index = buildEmbeddingIndex(scan, model);
+        const bytes = serializeEmbeddings(index);
+        writeFileSync(embedPath, bytes);
+        embedMeta = { embedVersion: EMBED_VERSION, modelId: model.modelId, sha1: sha1(bytes) };
+        embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+      }
+      // cache.json is written LAST so its meta always describes artifacts that
+      // are already on disk — a crash mid-way leaves stale meta whose shas
+      // fail the guard on the next run (safe: it just rebuilds).
+      writeCache({ graphSha1: sha1(graphJson), symbolsSha1: sha1(symbolsJson), embed: embedMeta });
+      process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
     }
-    process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
   } else if (cmd === "scan") {
-    const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     const summary = {
       engineVersion: ENGINE_VERSION,
       commit: scan.commit,
@@ -256,13 +408,13 @@ export async function runCli(argv: string[]): Promise<void> {
     };
     emit(JSON.stringify(summary, null, 2) + "\n", flags.out);
   } else if (cmd === "graph") {
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderGraphJson(graph), flags.out);
   } else if (cmd === "symbols") {
-    const { symbols } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { symbols } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderSymbolsJson(symbols), flags.out);
   } else if (cmd === "scip") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const bytes = renderScip(scan, { projectRoot: flags.projectRoot });
     const out = flags.out ?? resolve("index.scip");
     if (out === "-") process.stdout.write(Buffer.from(bytes));
@@ -271,14 +423,14 @@ export async function runCli(argv: string[]): Promise<void> {
       process.stderr.write(`codeindex: SCIP index → ${out} (${bytes.length} bytes)\n`);
     }
   } else if (cmd === "callers") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const index = buildCallerIndex(scan, undefined, { recall: flags.recall });
     const obj: Record<string, unknown> = {};
     for (const [name, entry] of index) obj[name] = entry;
     emit(JSON.stringify(obj, null, 2) + "\n", flags.out);
   } else if (cmd === "search") {
     if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     if (flags.semantic) {
       const endpoint = resolveEmbedEndpoint();
       const lexical = (): void => {
@@ -383,7 +535,7 @@ export async function runCli(argv: string[]): Promise<void> {
       }
       const model = loadEmbedModel(modelDir)!;
       mkdirSync(flags.out, { recursive: true });
-      const scan = scanRepo(flags.repo, scanOptions(flags));
+      const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
       const index = buildEmbeddingIndex(scan, model);
       writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index));
       process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`);
@@ -404,9 +556,14 @@ export async function runCli(argv: string[]): Promise<void> {
         return;
       }
       try {
-        JSON.parse(body);
-      } catch {
-        process.stderr.write("codeindex: pull failed — response is not a valid model.json (expected JSON)\n");
+        // Shape-validate BEFORE writing: a JSON-valid but shape-invalid asset
+        // would otherwise land on disk and turn every later semantic search
+        // into a hard loadEmbedModel error instead of the documented degrade.
+        parseEmbedModel(JSON.parse(body), url);
+      } catch (e) {
+        process.stderr.write(
+          `codeindex: pull failed — response is not a valid model.json (${e instanceof Error ? e.message : String(e)}) (nothing written)\n`,
+        );
         process.exitCode = 1;
         return;
       }
@@ -415,10 +572,109 @@ export async function runCli(argv: string[]): Promise<void> {
     } else {
       throw new Error("embed needs a subcommand: status | build | pull | serve");
     }
+  } else if (cmd === "grammars") {
+    const sub = flags.positional;
+    const cacheDir = sharedGrammarsCacheDir();
+    if (sub === "status") {
+      // Report which tier furnishes the wasms (adjacent/env/cache/none), the
+      // resolved dir, the pinned ENGINE_VERSION the cache is keyed on, and
+      // whether a pull is needed (no runtime wasm resolvable → AST off, regex).
+      const info = resolveGrammarsTier();
+      const runtimePresent = info.dir ? existsSync(join(info.dir, "web-tree-sitter.wasm")) : false;
+      const target = resolveGrammarsPullTarget();
+      const status: Record<string, unknown> = {
+        engineVersion: ENGINE_VERSION,
+        tier: info.tier,
+        dir: info.dir ?? null,
+        cacheDir,
+        runtimePresent,
+        pullNeeded: !runtimePresent,
+        url: target.url,
+      };
+      emit(JSON.stringify(status, null, 2) + "\n", flags.out);
+    } else if (sub === "pull") {
+      // Default: the official per-release asset + its `.sha256` sidecar. A
+      // user-set CODEINDEX_GRAMMARS_URL overrides both (mirror/custom, no
+      // verification). Fetch the expected digest from the sidecar first; a
+      // missing sidecar degrades to an unverified pull with a note (never fatal).
+      const target = resolveGrammarsPullTarget();
+      let expected: string | undefined;
+      if (target.sha256Url) {
+        try {
+          expected = await fetchExpectedSha256(target.sha256Url);
+        } catch (e) {
+          process.stderr.write(
+            `codeindex: could not fetch checksum (${e instanceof Error ? e.message : String(e)}) — proceeding unverified\n`,
+          );
+        }
+      }
+      // Idempotent: the marker sibling records the digest of the tarball that
+      // populated cacheDir. If the runtime wasm is present AND the marker matches
+      // the freshly-fetched digest, the cache is already up to date — skip the
+      // ~22 MB download entirely. (Keeps cacheDir itself byte-identical to the
+      // tarball; the marker lives next to it, never inside it.)
+      const runtime = join(cacheDir, "web-tree-sitter.wasm");
+      const markerPath = join(dirname(cacheDir), `${ENGINE_VERSION}.sha256`);
+      if (existsSync(runtime) && expected && existsSync(markerPath)) {
+        let marker = "";
+        try {
+          marker = readFileSync(markerPath, "utf8").trim();
+        } catch {
+          // unreadable marker → fall through and re-pull
+        }
+        if (marker === expected) {
+          process.stderr.write(`codeindex: grammars already present at ${cacheDir} (up to date)\n`);
+          return;
+        }
+      }
+      process.stderr.write(`codeindex: fetching grammars from ${target.url} → ${cacheDir}\n`);
+      let bytes: Uint8Array;
+      try {
+        // Follows redirects (GitHub → CDN) and verifies sha256 when known.
+        bytes = await fetchGrammarsTarball(target.url, expected);
+      } catch (e) {
+        process.stderr.write(`codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`);
+        process.exitCode = 1;
+        return;
+      }
+      // Atomic install: extract into a tmp dir SIBLING to cacheDir (same
+      // filesystem → rename is atomic), sanity-check the runtime wasm landed,
+      // then swap it into place. A failure at any step discards the tmp dir and
+      // leaves any existing cache untouched — a half-populated cache never shows.
+      let tmp: string | undefined;
+      try {
+        mkdirSync(dirname(cacheDir), { recursive: true });
+        tmp = mkdtempSync(join(dirname(cacheDir), ".grammars-tmp-"));
+        extractGrammarsTarball(bytes, tmp);
+        if (!existsSync(join(tmp, "web-tree-sitter.wasm"))) {
+          throw new Error("archive is missing web-tree-sitter.wasm");
+        }
+        if (existsSync(cacheDir)) rmSync(cacheDir, { recursive: true, force: true });
+        renameSync(tmp, cacheDir);
+        tmp = undefined;
+        if (expected) writeFileSync(markerPath, expected + "\n");
+      } catch (e) {
+        if (tmp) {
+          try {
+            rmSync(tmp, { recursive: true, force: true });
+          } catch {
+            // best-effort cleanup
+          }
+        }
+        process.stderr.write(
+          `codeindex: pull failed — ${e instanceof Error ? e.message : String(e)} (nothing written)\n`,
+        );
+        process.exitCode = 1;
+        return;
+      }
+      process.stderr.write(`codeindex: grammars extracted → ${cacheDir}\n`);
+    } else {
+      throw new Error("grammars needs a subcommand: status | pull");
+    }
   } else if (cmd === "rules") {
     if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
     const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8")));
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     const violations = checkRules(graph, rules);
     const errors = violations.filter((v) => v.severity === "error").length;
     emit(JSON.stringify({ errors, warnings: violations.length - errors, violations }, null, 2) + "\n", flags.out);
@@ -439,26 +695,26 @@ export async function runCli(argv: string[]): Promise<void> {
     for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!;
     emit(JSON.stringify({ ok, churn: sorted }, null, 2) + "\n", flags.out);
   } else if (cmd === "repomap") {
-    const { scan, graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { scan, graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderRepoMap(scan, graph, { budgetTokens: flags.budgetTokens }), flags.out);
   } else if (cmd === "hotspots") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const { churn, ok } = gitChurn(flags.repo, { since: flags.since });
     emit(JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2) + "\n", flags.out);
   } else if (cmd === "coupling") {
     const { ok, couplings } = changeCoupling(flags.repo, { since: flags.since });
     emit(JSON.stringify({ ok, couplings }, null, 2) + "\n", flags.out);
   } else if (cmd === "deadcode") {
-    emit(JSON.stringify(findDeadCode(scanRepo(flags.repo, scanOptions(flags))), null, 2) + "\n", flags.out);
+    emit(JSON.stringify(findDeadCode(scanRepo(flags.repo, scanOptions(flags, precomputedWalk))), null, 2) + "\n", flags.out);
   } else if (cmd === "complexity") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     emit(JSON.stringify(symbolComplexity(scan, flags.positional), null, 2) + "\n", flags.out);
   } else if (cmd === "risk") {
-    const scan = scanRepo(flags.repo, scanOptions(flags));
+    const scan = scanRepo(flags.repo, scanOptions(flags, precomputedWalk));
     const { churn, ok } = gitChurn(flags.repo, { since: flags.since });
     emit(JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2) + "\n", flags.out);
   } else if (cmd === "mermaid") {
-    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags));
+    const { graph } = buildIndexArtifacts(flags.repo, scanOptions(flags, precomputedWalk));
     emit(renderMermaid(graph, { module: flags.positional }), flags.out);
   } else if (cmd === "grep") {
     if (!flags.positional) throw new Error("grep needs a pattern: cli.mjs grep <pattern> --repo <dir>");

package/src/engine.ts

@@ -45,8 +45,23 @@ export { extractMarkdown } from "./extract/markdown.js";
 export type { MarkdownInfo } from "./extract/markdown.js";
 
 // AST tier (optional — a no-op without the grammar wasm sidecar).
-export { ensureGrammars, allGrammarKeys, grammarKeyForExt, grammarReady } from "./ast/loader.js";
+export { ensureGrammars, allGrammarKeys, grammarKeysForExts, grammarKeyForExt, grammarReady } from "./ast/loader.js";
 export { extractAst } from "./ast/extract.js";
+// Grammars resolution + the slim pull/cache tier (v2.14.0): resolve the wasm
+// dir across bundle-adjacent → env → shared cache → none, and fetch the
+// committed wasms into the shared cache when none ship next to the bundle
+// (`codeindex grammars pull|status`). Additive; opt-in; offline-safe.
+export { resolveGrammarsDir, resolveGrammarsTier, sharedGrammarsCacheDir } from "./ast/loader.js";
+export type { GrammarsTier, GrammarsTierName } from "./ast/loader.js";
+export {
+  DEFAULT_GRAMMARS_URL,
+  resolveGrammarsPullTarget,
+  fetchGrammarsTarball,
+  fetchExpectedSha256,
+  extractTarInto,
+  extractGrammarsTarball,
+} from "./ast/grammars-pull.js";
+export type { GrammarsPullTarget } from "./ast/grammars-pull.js";
 
 // Resolution + modules + graph tier.
 export { buildResolveContext, resolveImport, resolveDocLink } from "./resolve.js";
@@ -78,8 +93,9 @@ export { renderGraphJson } from "./render/graph-json.js";
 export { renderScip } from "./render/scip.js";
 export type { RenderScipOptions } from "./render/scip.js";
 
-// One-call pipeline.
-export { buildIndexArtifacts } from "./pipeline.js";
+// One-call pipeline (buildArtifactsFromScan: the scan-onward half, for callers
+// that already hold a RepoScan).
+export { buildIndexArtifacts, buildArtifactsFromScan } from "./pipeline.js";
 export type { BuildIndexOptions, IndexArtifacts } from "./pipeline.js";
 
 // Git utilities.
@@ -114,7 +130,7 @@ export {
   loadEmbedModel,
   resolveEmbedPullUrl,
 } from "./embed/model.js";
-export type { StaticEmbedModel } from "./embed/model.js";
+export type { StaticEmbedModel, EmbedPullTarget } from "./embed/model.js";
 export { encode, quantize, tokenize, wordpiece, basicTokenize, roundHalfToEven, intDot } from "./embed/encode.js";
 export { buildEmbeddingIndex, serializeEmbeddings, deserializeEmbeddings, embeddingUnits } from "./embed/index.js";
 export type { EmbeddingIndex, EmbeddingRecord, EmbeddingUnit } from "./embed/index.js";
@@ -161,6 +177,7 @@ export type { RepoMapOptions } from "./repomap.js";
 
 // MCP server over stdio (also reachable as `engine.mjs mcp`).
 export { runMcpServer } from "./mcp.js";
+export type { McpServerOptions } from "./mcp.js";
 
 // General-purpose helpers shared by consumers (deterministic, dependency-free).
 export { sha1, shortHash } from "./hash.js";

package/src/extract/code.ts

@@ -302,12 +302,13 @@ const DEF_INTRODUCERS = /(?:\bfunction|\bdef|\bfunc|\bfun|\bfn|\bclass|\bsub|\bm
 export function collectCallsRegex(
   content: string,
   symbols: Pick<CodeSymbol, "name" | "line">[] = [],
+  maxCalls: number = 512,
 ): { name: string; line: number; receiver?: string }[] {
   const out = new Map<string, { name: string; line: number; receiver?: string }>();
   const ownDefLines = new Set(symbols.map((s) => `${s.name} ${s.line}`));
   const lines = content.split("\n");
   const CALL_RE = /(?:\bnew\s+)?(?:([A-Za-z_$][\w$]*)\s*\.\s*)?([A-Za-z_$][\w$]*)\s*\(/g;
-  for (let i = 0; i < lines.length && out.size < 512; i++) {
+  for (let i = 0; i < lines.length && out.size < maxCalls; i++) {
     const line = lines[i]!;
     // Cheap comment guard: a line-leading comment marker means no calls here
     // (block-comment interiors and strings stay best-effort, like the symbol
@@ -332,7 +333,7 @@ export function collectCallsRegex(
     CALL_RE.lastIndex = 0;
     let m: RegExpExecArray | null;
     const fallbackExcluded = new Set<string>();
-    while ((m = CALL_RE.exec(line)) !== null && out.size < 512) {
+    while ((m = CALL_RE.exec(line)) !== null && out.size < maxCalls) {
       const receiver = m[1];
       const name = m[2]!;
       if (name.length < 2 || CALL_KEYWORDS.has(name)) continue;
@@ -350,13 +351,16 @@ export function collectCallsRegex(
   return [...out.values()].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : a.line - b.line));
 }
 
-export function extractCode(rel: string, ext: string, content: string): CodeInfo {
+// `opts.maxCallsPerFile` overrides the per-file call-site cap (default 512) on
+// BOTH extraction tiers — AST and regex — so recall-oriented consumers can raise
+// it. Dedup/sort semantics are unchanged; absent, output is byte-identical.
+export function extractCode(rel: string, ext: string, content: string, opts: { maxCallsPerFile?: number } = {}): CodeInfo {
   // Symbols come from tree-sitter when a grammar is loaded for this extension
   // (AST-exact: real nesting, precise kinds, structural export), else the regex
   // extractors. Imports/pkg stay on the battle-tested regex path here — their
   // resolution is covered by resolve tests and the e2e ratchet; the new-language
   // AST importers land with their resolvers.
-  const ast = extractAst(rel, ext, content);
+  const ast = extractAst(rel, ext, content, { maxCalls: opts.maxCallsPerFile });
   const symbols = (ast ? ast.symbols : extractSymbols(rel, ext, content)).slice(0, 400);
   // Add barrel re-exports the local def didn't already cover.
   const known = new Set(symbols.map((s) => s.name));
@@ -378,7 +382,7 @@ export function extractCode(rel: string, ext: string, content: string): CodeInfo
     // collector otherwise, so caller indexes exist without the wasm sidecar.
     // `symbols` (this file's own regex-extracted defs) lets the collector
     // exclude a definition's own name+line from its call candidates.
-    calls: ast ? ast.calls : collectCallsRegex(content, symbols),
+    calls: ast ? ast.calls : collectCallsRegex(content, symbols, opts.maxCallsPerFile),
     importedNames: ast?.importedNames,
   };
 }

package/src/hash.ts

@@ -2,8 +2,11 @@ import { createHash } from "node:crypto";
 
 // Stable content hash used for the manifest's staleness oracle and for the
 // `ui:gen` region fingerprints. sha1 is plenty for change detection (not
-// security) and keeps the manifest compact.
-export function sha1(s: string): string {
+// security) and keeps the manifest compact. Accepts raw bytes too (additive)
+// so binary artifacts (e.g. embeddings.bin) hash without a lossy decode; a
+// string is hashed as its UTF-8 bytes — identical to hashing the bytes
+// writeFileSync would put on disk for that string.
+export function sha1(s: string | Uint8Array): string {
   return createHash("sha1").update(s).digest("hex");
 }
 

package/src/lang/js-ts.ts

@@ -7,7 +7,7 @@ import { scan, type Rule } from "./common.js";
 const RULES: Rule[] = [
   { re: /^\s*export\s+(?:async\s+)?function\s+(?<name>[\w$]+)/, kind: "function", exported: true },
   { re: /^\s*export\s+default\s+(?:async\s+)?function\s+(?<name>[\w$]+)/, kind: "function", exported: true },
-  { re: /^\s*export\s+default\s+(?:abstract\s+)?class\s+(?<name>[\w$]+)/, kind: "class", exported: true },
+  { re: /^\s*export\s+default\s+(?:abstract\s+)?class\s+(?!extends\b)(?<name>[\w$]+)/, kind: "class", exported: true },
   { re: /^\s*(?:async\s+)?function\s+(?<name>[\w$]+)/, kind: "function", exported: false },
   { re: /^\s*export\s+(?:abstract\s+)?class\s+(?<name>[\w$]+)/, kind: "class", exported: true },
   { re: /^\s*(?:abstract\s+)?class\s+(?<name>[\w$]+)/, kind: "class", exported: false },