@maxgfr/codeindex 2.12.0 → 2.14.0

package/src/ast/extract.ts

@@ -399,8 +399,9 @@ function readReceiver(node: TSNode | null): string | undefined {
 // dedicated member-call node's `name`; "constructor" reads the constructed type.
 // A qualified call also carries the immediate `receiver` name (see readReceiver).
 // Names are filtered to plausible identifiers (≥ 2 chars), deduped by name+line,
-// sorted, and capped, so the set stays small and deterministic.
-function collectCalls(root: TSNode, spec: LangSpec): { name: string; line: number; receiver?: string }[] {
+// sorted, and capped (default MAX_CALLS; overridable via maxCalls), so the set
+// stays small and deterministic.
+function collectCalls(root: TSNode, spec: LangSpec, maxCalls: number = MAX_CALLS): { name: string; line: number; receiver?: string }[] {
   if (!spec.calls) return [];
   const out: { name: string; line: number; receiver?: string }[] = [];
   const seen = new Set<string>();
@@ -439,7 +440,7 @@ function collectCalls(root: TSNode, spec: LangSpec): { name: string; line: numbe
   };
   visit(root);
   out.sort((a, b) => byStr(a.name, b.name) || a.line - b.line);
-  return out.slice(0, MAX_CALLS);
+  return out.slice(0, maxCalls);
 }
 
 // Collect JS/TS named-import bindings: `import { a, b as c } from "x"` →
@@ -476,7 +477,8 @@ function collectImportedNames(root: TSNode, spec: LangSpec): string[] {
 // Extract declared symbols from one file via its committed grammar. Returns
 // undefined when no grammar is loaded for the extension (caller falls back to the
 // regex extractor). Walks top-level declarations plus one level of nested members.
-export function extractAst(rel: string, ext: string, content: string): AstResult | undefined {
+// `opts.maxCalls` overrides the per-file call-site cap (default MAX_CALLS).
+export function extractAst(rel: string, ext: string, content: string, opts: { maxCalls?: number } = {}): AstResult | undefined {
   const key = grammarKeyForExt(ext);
   if (!key || !grammarReady(key)) return undefined;
   const spec = SPECS[key];
@@ -697,7 +699,7 @@ export function extractAst(rel: string, ext: string, content: string): AstResult
 
     const refs = collectImports(root, spec);
     const idents = collectRefIdents(root, new Set(symbols.map((s) => s.name)));
-    const calls = collectCalls(root, spec);
+    const calls = collectCalls(root, spec, opts.maxCalls);
     const importedNames = collectImportedNames(root, spec);
     let pkg: string | undefined;
     if (spec.lang === "java") {

package/src/ast/grammars-pull.ts

@@ -0,0 +1,178 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve, sep } from "node:path";
+import { gunzipSync } from "node:zlib";
+import { ENGINE_VERSION } from "../types.js";
+
+// The slim grammars-pull tier (v2.14.0). The tree-sitter wasm sidecar
+// (scripts/grammars/, ~22 MB) is OPTIONAL and opt-in by presence, exactly like
+// the embedding model (src/embed/model.ts): no wasm resolvable anywhere → the
+// engine silently stays on the regex tier (src/ast/loader.ts). This module lets
+// a consumer that vendors ONLY the zero-dependency engine.mjs (no adjacent
+// grammars/) fetch the same committed wasms into a shared, version-scoped cache
+// — `codeindex grammars pull` — and get byte-identical AST extraction from the
+// cache as from a bundle-adjacent dir (same wasm bytes → same AST → same
+// symbols). NOTHING here runs during indexing; a failed/absent pull only ever
+// leaves the cache empty, never throws into the scan.
+//
+// Zero runtime dependencies: global fetch (Node ≥18, follows GitHub → CDN
+// redirects — mirrors `embed pull`), node:crypto for sha256, node:zlib for
+// gunzip, and a tiny inline ustar reader below (Node has no built-in tar). We
+// deliberately do NOT spawn the system `tar`: the embed-pull precedent is pure
+// Node stdlib, and an in-process reader is cross-platform-deterministic and
+// lets us guard path traversal precisely before a single byte hits disk.
+
+// The official grammars asset, published as a normal release asset on the
+// engine's own `v<ENGINE_VERSION>` tag (built + uploaded by the release
+// workflow, per-version). `grammars pull` uses this when CODEINDEX_GRAMMARS_URL
+// is unset.
+export const DEFAULT_GRAMMARS_URL = `https://github.com/maxgfr/codeindex/releases/download/v${ENGINE_VERSION}/grammars-${ENGINE_VERSION}.tar.gz`;
+
+// What `grammars pull` fetches, and how to verify it. CODEINDEX_GRAMMARS_URL
+// wins outright (the user's explicit override / private mirror) and carries NO
+// checksum, so a custom asset keeps the un-verified behavior — exactly like
+// resolveEmbedPullUrl's custom-mirror rule. With no env we fall back to the
+// built-in default AND its `.sha256` SIDECAR asset. Unlike the embed model (a
+// single fixed release whose hash is pinned as a code constant), the grammars
+// tarball is cut fresh on EVERY release, so its hash cannot be a compile-time
+// constant; the checksum travels next to the asset and is fetched at pull time.
+export interface GrammarsPullTarget {
+  url: string;
+  sha256Url?: string; // present only for the built-in default → fetch + verify
+}
+
+export function resolveGrammarsPullTarget(): GrammarsPullTarget {
+  const env = process.env.CODEINDEX_GRAMMARS_URL;
+  if (env && env.trim()) return { url: env.trim() };
+  return { url: DEFAULT_GRAMMARS_URL, sha256Url: `${DEFAULT_GRAMMARS_URL}.sha256` };
+}
+
+// Fetch the tarball bytes over HTTP, following redirects. When `expectedSha256`
+// is given, the downloaded bytes are hashed and MUST match or this throws with
+// a clear message (so a corrupt/tampered default asset never reaches disk). A
+// non-2xx response also throws. I/O + verification are split from the CLI so
+// both the success and the sha-mismatch paths are unit-testable — mirrors
+// fetchEmbedModel.
+export async function fetchGrammarsTarball(url: string, expectedSha256?: string): Promise<Uint8Array> {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
+  const buf = Buffer.from(await res.arrayBuffer());
+  if (expectedSha256) {
+    const got = createHash("sha256").update(buf).digest("hex");
+    if (got !== expectedSha256) {
+      throw new Error(`sha256 mismatch: expected ${expectedSha256}, got ${got}`);
+    }
+  }
+  return buf;
+}
+
+// Wrap any Uint8Array as a Buffer VIEW over the same memory (no copy) so the
+// tar reader can use Buffer's string decoders. Keeps the public API on
+// Uint8Array — the engine's convention — while avoiding a Node-only `Buffer`
+// type in the shipped .d.mts (consumers may lack @types/node).
+function asBuffer(u: Uint8Array): Buffer {
+  return Buffer.isBuffer(u) ? u : Buffer.from(u.buffer, u.byteOffset, u.byteLength);
+}
+
+// Fetch + parse the `.sha256` sidecar (a `<hex>` or `<hex>  <filename>` line),
+// returning the lowercased 64-hex digest. Throws on a non-2xx or a malformed
+// body so a bad sidecar degrades the pull to a clear error rather than silently
+// skipping verification.
+export async function fetchExpectedSha256(url: string): Promise<string> {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
+  const text = await res.text();
+  const hex = (text.trim().split(/\s+/)[0] ?? "").toLowerCase();
+  if (!/^[0-9a-f]{64}$/.test(hex)) throw new Error(`invalid sha256 sidecar at ${url}`);
+  return hex;
+}
+
+// One parsed tar entry (header fields we care about + its raw data slice).
+interface TarEntry {
+  name: string;
+  type: string; // typeflag char: "0"/"\0" = regular file, "5" = dir, …
+  data: Buffer;
+}
+
+// Read a C-string (NUL-terminated, else the whole field) from a 512-byte header.
+function cstr(block: Buffer, start: number, len: number): string {
+  const slice = block.subarray(start, start + len);
+  const nul = slice.indexOf(0);
+  return slice.toString("utf8", 0, nul === -1 ? slice.length : nul);
+}
+
+// A minimal ustar/gnu reader — enough for the flat, short-named wasm archive we
+// publish (no long-name/PAX/sparse entries). Yields every entry in order; the
+// caller decides which to keep. Stops at the end-of-archive zero block.
+function* readTar(buf: Buffer): Generator<TarEntry> {
+  let off = 0;
+  while (off + 512 <= buf.length) {
+    const block = buf.subarray(off, off + 512);
+    let allZero = true;
+    for (let i = 0; i < 512; i++) {
+      if (block[i] !== 0) {
+        allZero = false;
+        break;
+      }
+    }
+    if (allZero) break; // end-of-archive marker
+    const name = cstr(block, 0, 100);
+    const prefix = cstr(block, 345, 155); // ustar long-path prefix (usually empty)
+    const sizeStr = cstr(block, 124, 12).trim();
+    const size = sizeStr ? parseInt(sizeStr, 8) : 0;
+    const type = String.fromCharCode(block[156] ?? 0);
+    off += 512;
+    const data = buf.subarray(off, off + size);
+    off += Math.ceil(size / 512) * 512;
+    yield { name: prefix ? `${prefix}/${name}` : name, type, data };
+  }
+}
+
+// Normalize a tar entry name to a SAFE relative path, or null to reject it.
+// Rejects absolute paths (POSIX "/…", Windows "C:…" or "\…"), any ".." segment,
+// and embedded NULs; drops "." and empty segments. This is the path-traversal
+// guard: a "../../etc/x" or "/etc/x" entry can never escape the destination.
+function safeRelPath(name: string): string | null {
+  if (!name || name.includes("\0")) return null;
+  if (name.startsWith("/") || name.startsWith("\\") || /^[A-Za-z]:/.test(name)) return null;
+  const out: string[] = [];
+  for (const part of name.split(/[/\\]/)) {
+    if (part === "" || part === ".") continue;
+    if (part === "..") return null;
+    out.push(part);
+  }
+  return out.length ? out.join("/") : null;
+}
+
+// Extract every regular file from a RAW (already-gunzipped) tar into destDir,
+// returning the relative paths written. Throws on the FIRST unsafe entry BEFORE
+// writing it (defense in depth: also verifies the resolved path stays within
+// destDir). Non-regular entries (dirs, symlinks, headers) are skipped. Callers
+// extract into a throwaway tmp dir, so a mid-stream throw leaves nothing
+// half-installed once that tmp dir is discarded.
+export function extractTarInto(rawTar: Uint8Array, destDir: string): string[] {
+  const root = resolve(destDir);
+  const written: string[] = [];
+  for (const entry of readTar(asBuffer(rawTar))) {
+    if (entry.type !== "0" && entry.type !== "\0") continue; // regular files only
+    const rel = safeRelPath(entry.name);
+    if (rel === null) throw new Error(`refusing unsafe tar entry: ${entry.name}`);
+    const dest = resolve(destDir, rel);
+    if (dest !== root && !dest.startsWith(root + sep)) {
+      throw new Error(`tar entry escapes destination: ${entry.name}`);
+    }
+    mkdirSync(dirname(dest), { recursive: true });
+    writeFileSync(dest, entry.data);
+    written.push(rel);
+  }
+  return written;
+}
+
+// gunzip (when the bytes are gzip-framed) then extract. The default asset is a
+// `.tar.gz`; a custom CODEINDEX_GRAMMARS_URL serving a plain `.tar` is handled
+// too (magic-byte sniff), so the tier does not force a compression format.
+export function extractGrammarsTarball(bytes: Uint8Array, destDir: string): string[] {
+  const b = asBuffer(bytes);
+  const raw = b.length >= 2 && b[0] === 0x1f && b[1] === 0x8b ? gunzipSync(b) : b;
+  return extractTarInto(raw, destDir);
+}

package/src/ast/loader.ts

@@ -1,7 +1,9 @@
 import { readFileSync, existsSync } from "node:fs";
+import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { Parser, Language } from "web-tree-sitter";
+import { ENGINE_VERSION } from "../types.js";
 
 // Extension → committed grammar wasm key (scripts/grammars/<key>.wasm). Only the
 // languages we ship a grammar for appear here; everything else falls back to the
@@ -28,23 +30,65 @@ export function grammarKeyForExt(ext: string): string | undefined {
   return EXT_GRAMMAR[ext];
 }
 
-// Where the committed wasms live. Resolved relative to this module so the
-// SAME logic works whether we run from the tsup bundle (scripts/engine.mjs →
-// scripts/grammars), from a consumer's vendored copy (src/vendor →
-// ../../scripts/grammars), or from source under vitest (src/ast →
-// ../../scripts/grammars). CODEINDEX_GRAMMAR_DIR overrides for tests/tooling
-// (ULTRAINDEX_GRAMMAR_DIR kept as a legacy alias).
-function resolveGrammarDir(): string {
-  const env = process.env.CODEINDEX_GRAMMAR_DIR ?? process.env.ULTRAINDEX_GRAMMAR_DIR;
-  if (env && existsSync(env)) return env;
-  const here = dirname(fileURLToPath(import.meta.url));
-  const candidates = [
+// Which supplier furnished the resolved grammars dir. Reported by
+// `codeindex grammars status`; "none" is the regex-tier signal.
+export type GrammarsTierName = "adjacent" | "env" | "cache" | "none";
+
+export interface GrammarsTier {
+  tier: GrammarsTierName;
+  dir?: string; // undefined only when tier === "none"
+  cacheDir: string; // where a `grammars pull` would extract, regardless of tier
+}
+
+// The shared, version-scoped cache a `grammars pull` extracts into and the
+// resolver falls back to when no wasm ships next to the bundle. Version-scoped
+// so a consumer bumping the engine never loads a stale grammar set: a new
+// ENGINE_VERSION points at a fresh, empty dir until its own pull runs (and the
+// old dir's bytes remain byte-identical to what that engine shipped). Honors
+// XDG_CACHE_HOME, else ~/.cache — the platform-neutral, dependency-free
+// convention already used by the wider toolchain.
+export function sharedGrammarsCacheDir(): string {
+  const xdg = process.env.XDG_CACHE_HOME;
+  const base = xdg && xdg.trim() ? xdg.trim() : join(homedir(), ".cache");
+  return join(base, "codeindex", "grammars", ENGINE_VERSION);
+}
+
+// Resolve the grammars dir AND record which tier supplied it, IN ORDER:
+//   1. an explicit CODEINDEX_GRAMMAR_DIR / ULTRAINDEX_GRAMMAR_DIR override
+//      (legacy, singular; kept winning outright so vendored/test setups that
+//      pin it behave exactly as before) — reported as the "env" tier;
+//   2. (a) the bundle-adjacent grammars/ dir — the shipped default: works from
+//      the tsup bundle (scripts/engine.mjs → scripts/grammars), a consumer's
+//      vendored copy (src/vendor → ../../scripts/grammars) or source under
+//      vitest (src/ast → ../../scripts/grammars). Wins if present, so the
+//      offline, no-network story is untouched;
+//   3. (b) CODEINDEX_GRAMMARS_DIR — an explicit shared/custom dir override;
+//   4. (c) the shared version-scoped cache a `grammars pull` populates;
+//   5. (d) nothing resolvable → tier "none", dir undefined → the regex tier.
+// Never touches the network and never throws. `moduleDir` overrides the
+// module-relative base of the bundle-adjacent probe (tests/tooling only).
+export function resolveGrammarsTier(opts: { moduleDir?: string } = {}): GrammarsTier {
+  const cacheDir = sharedGrammarsCacheDir();
+  const legacy = process.env.CODEINDEX_GRAMMAR_DIR ?? process.env.ULTRAINDEX_GRAMMAR_DIR;
+  if (legacy && legacy.trim() && existsSync(legacy)) return { tier: "env", dir: legacy, cacheDir };
+  const here = opts.moduleDir ?? dirname(fileURLToPath(import.meta.url));
+  const adjacent = [
     join(here, "grammars"), // bundle: <...>/scripts/grammars
     join(here, "..", "..", "scripts", "grammars"), // dev: src/ast → <repo>/scripts/grammars
     join(here, "..", "scripts", "grammars"),
   ];
-  for (const c of candidates) if (existsSync(c)) return c;
-  return join(here, "grammars");
+  for (const c of adjacent) if (existsSync(c)) return { tier: "adjacent", dir: c, cacheDir };
+  const env = process.env.CODEINDEX_GRAMMARS_DIR;
+  if (env && env.trim() && existsSync(env)) return { tier: "env", dir: env, cacheDir };
+  if (existsSync(cacheDir)) return { tier: "cache", dir: cacheDir, cacheDir };
+  return { tier: "none", cacheDir };
+}
+
+// The chosen grammars dir, or undefined when nothing is resolvable anywhere
+// (the caller then stays on the regex tier). Additive companion to
+// resolveGrammarsTier — same resolution, dir only.
+export function resolveGrammarsDir(opts?: { moduleDir?: string }): string | undefined {
+  return resolveGrammarsTier(opts).dir;
 }
 
 // tree-sitter's runtime + grammars must be initialised asynchronously (wasm
@@ -62,10 +106,11 @@ const failed = new Set<string>();
 // and safe to call repeatedly. A missing/broken wasm is remembered as failed so
 // the caller silently falls back to regex rather than retrying every file.
 export async function ensureGrammars(keys: Iterable<string>): Promise<void> {
-  const dir = resolveGrammarDir();
+  const dir = resolveGrammarsDir();
+  if (!dir) return; // nothing resolvable (adjacent/env/cache all absent) → regex everywhere
   if (!runtimeReady) {
     const runtime = join(dir, "web-tree-sitter.wasm");
-    if (!existsSync(runtime)) return; // no committed grammars → regex fallback everywhere
+    if (!existsSync(runtime)) return; // dir present but no runtime wasm → regex fallback everywhere
     await Parser.init({ wasmBinary: readFileSync(runtime) as unknown as Uint8Array });
     runtimeReady = true;
     parser = new Parser();
@@ -90,6 +135,23 @@ export function allGrammarKeys(): string[] {
   return [...new Set(Object.values(EXT_GRAMMAR))];
 }
 
+// The grammar keys needed for a set of file extensions: each mapped through
+// EXT_GRAMMAR, unknown extensions dropped, then deduped and sorted. Warming
+// exactly this set (instead of every committed grammar) skips the wasm load for
+// languages the repo doesn't contain, while keeping output byte-identical:
+// extractAst falls back to regex only when grammarReady(key) is false, and the
+// walk's extension set (which feeds this) is a superset of what scanRepo keeps,
+// so every extracted file has its grammar loaded. Language.load calls are
+// independent, so loading fewer grammars cannot change parses of loaded ones.
+export function grammarKeysForExts(exts: Iterable<string>): string[] {
+  const keys = new Set<string>();
+  for (const ext of exts) {
+    const key = EXT_GRAMMAR[ext];
+    if (key !== undefined) keys.add(key);
+  }
+  return [...keys].sort();
+}
+
 export function grammarReady(key: string): boolean {
   return loaded.has(key);
 }

package/src/bm25.ts

@@ -17,6 +17,7 @@
 // query byte-identical: the expansion only ever engages on terms that would
 // otherwise contribute nothing.
 import type { RepoScan } from "./scan.js";
+import { bm25DocsFor, bm25TrigramsFor } from "./derived.js";
 import { foldText, keywords } from "./util.js";
 import { byStr } from "./sort.js";
 
@@ -72,7 +73,10 @@ export function subtokens(raw: string): string[] {
   return out;
 }
 
-interface Doc {
+// Exported for src/derived.ts's per-scan cache only — NOT part of the public
+// barrel (engine.ts). searchIndex reads docs strictly, never mutates them, so
+// the cached array/objects can be shared across calls on the same scan.
+export interface Doc {
   file: string;
   tf: Map<string, number>;
   len: number; // total token occurrences (the BM25 length normalizer)
@@ -86,7 +90,8 @@ function addTerms(doc: Doc, text: string): void {
   }
 }
 
-function buildDocs(scan: RepoScan): Doc[] {
+// Exported for src/derived.ts (bm25DocsFor) — not in the public barrel.
+export function buildDocs(scan: RepoScan): Doc[] {
   const docs: Doc[] = [];
   for (const f of scan.files) {
     const doc: Doc = { file: f.rel, tf: new Map(), len: 0, symbols: [] };
@@ -128,8 +133,9 @@ export function diceCoefficient(a: ReadonlySet<string>, b: ReadonlySet<string>):
 
 // Trigram index of the corpus vocabulary: every distinct doc token mapped to
 // its trigram set. Built LAZILY by searchIndex — only when >=1 query term has
-// df==0 — so a fully-matched query never pays this cost.
-function buildTrigramIndex(docs: Doc[]): Map<string, Set<string>> {
+// df==0 — so a fully-matched query never pays this cost. Exported for
+// src/derived.ts (bm25TrigramsFor) — not in the public barrel.
+export function buildTrigramIndex(docs: Doc[]): Map<string, Set<string>> {
   const index = new Map<string, Set<string>>();
   for (const d of docs) {
     for (const term of d.tf.keys()) {
@@ -155,7 +161,9 @@ export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions =
   }
   if (!terms.length) return [];
 
-  const docs = buildDocs(scan);
+  // Memoized per scan (src/derived.ts) — identical docs to a direct build;
+  // read-only from here on.
+  const docs = bm25DocsFor(scan);
   const n = docs.length;
   if (!n) return [];
   let totalLen = 0;
@@ -179,7 +187,9 @@ export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions =
   if (fuzzyEnabled) {
     const unmatched = terms.filter((t) => df.get(t) === 0);
     if (unmatched.length) {
-      const trigramIndex = buildTrigramIndex(docs);
+      // Still lazy (only reached on a zero-df term), now also cached per scan:
+      // the first fuzzy query builds the vocabulary trigrams, later ones reuse.
+      const trigramIndex = bm25TrigramsFor(scan);
       for (const t of unmatched) {
         const grams = charTrigrams(t);
         const candidates: { term: string; dice: number }[] = [];

package/src/complexity.ts

@@ -5,6 +5,7 @@
 // changed are where defects concentrate.
 import { join } from "node:path";
 import type { RepoScan } from "./scan.js";
+import { fileComplexityFor } from "./derived.js";
 import { readText } from "./walk.js";
 import { byStr } from "./sort.js";
 
@@ -57,10 +58,15 @@ export interface RiskHotspot {
 }
 
 export function riskHotspots(scan: RepoScan, churn: Map<string, number>, top = 20): RiskHotspot[] {
+  // Per-file branch counts memoized per scan (src/derived.ts): the first call
+  // still reads every code file from disk, repeat calls become map lookups.
+  // fileComplexityFor covers exactly the code files filtered below, so the
+  // lookup always hits.
+  const complexityByFile = fileComplexityFor(scan);
   const out: RiskHotspot[] = scan.files
     .filter((f) => f.kind === "code")
     .map((f) => {
-      const complexity = complexityOfSource(readText(join(scan.root, f.rel)));
+      const complexity = complexityByFile.get(f.rel)!;
       const commits = churn.get(f.rel) ?? 0;
       return { file: f.rel, complexity, commits, score: (commits + 1) * complexity };
     });

package/src/deadcode.ts

@@ -7,8 +7,7 @@
 // consumed externally), not by substring matching.
 import type { CodeSymbol } from "./types.js";
 import type { RepoScan } from "./scan.js";
-import { buildCallerIndex } from "./callers.js";
-import { computeSymbolRefs } from "./render/symbols-json.js";
+import { callerIndexFor, symbolRefsFor } from "./derived.js";
 import { isTestPath } from "./tests-map.js";
 import { byStr } from "./sort.js";
 
@@ -26,8 +25,10 @@ export interface DeadSymbol {
 }
 
 export function findDeadCode(scan: RepoScan): DeadSymbol[] {
-  const callers = buildCallerIndex(scan);
-  const refs = computeSymbolRefs(scan);
+  // Memoized per scan (src/derived.ts) and READ-ONLY here — this function
+  // only .get()s from both structures, so sharing the cached objects is safe.
+  const callers = callerIndexFor(scan);
+  const refs = symbolRefsFor(scan);
   const out: DeadSymbol[] = [];
   const consider = (s: CodeSymbol): boolean =>
     s.exported && !REFERENCE_KINDS.has(s.kind) && !isTestPath(s.file) && !ENTRYPOINT_RE.test(s.file);

package/src/derived.ts

@@ -0,0 +1,128 @@
+// Per-scan derived-structure cache (INTERNAL — deliberately NOT exported from
+// the src/engine.ts barrel). Several public entry points independently
+// recompute the same expensive structures from the same RepoScan object:
+// findReferences, findDeadCode and the pipeline each rebuild the caller index
+// / symbol refs; computeImportPairs rebuilds the resolve context; searchIndex
+// rebuilds the BM25 documents; riskHotspots re-reads every code file. This
+// module memoizes those structures once per scan, keyed by RepoScan object
+// identity in a WeakMap — a scan's cache lives exactly as long as the scan
+// itself, and two scans (even of the same repo) never share entries.
+//
+// Determinism: every builder memoized here is a pure function of the scan
+// (plus, for the resolve context and per-file complexity, the on-disk bytes
+// the scan describes), so a cache hit returns exactly what a recompute would —
+// artifacts stay byte-identical; only repeated work disappears.
+//
+// Sharing contract: accessors return the SAME object on every call for a
+// given scan. Callers must treat the results as READ-ONLY and copy anything
+// they hand to a consumer that owns its result (findReferences copies
+// callSites; computeImportPairs returns a fresh Set) — a consumer mutation
+// reaching a cached object would poison every later query on that scan.
+//
+// Only the DEFAULT (precision) caller index is memoized: recall-mode variants
+// take options, and buildCallerIndex's public path keeps building a fresh
+// index per call (it only shares the memoized import pairs).
+//
+// Import cycles: callers.ts / bm25.ts / complexity.ts import their accessor
+// from here while this module imports their builders — function-level cycles
+// only (no module-evaluation-time cross-calls), which Node ESM and esbuild
+// resolve safely.
+import { join } from "node:path";
+import type { RepoScan } from "./scan.js";
+import { buildResolveContext, resolveImport, type ResolveContext } from "./resolve.js";
+import { uniqueSymbolDefs } from "./graph.js";
+import { computeSymbolRefs } from "./render/symbols-json.js";
+import { buildCallerIndex, type CallerIndex } from "./callers.js";
+import { buildDocs, buildTrigramIndex, type Doc } from "./bm25.js";
+import { complexityOfSource } from "./complexity.js";
+import { readText } from "./walk.js";
+
+interface DerivedCache {
+  resolveCtx?: ResolveContext;
+  importPairs?: Set<string>; // `${from}|${to}` resolved-import pairs
+  uniqueDefs?: Map<string, string>; // uniqueSymbolDefs(scan)
+  symbolRefs?: Map<string, Set<string>>; // computeSymbolRefs(scan)
+  callerIndex?: CallerIndex; // DEFAULT precision opts only — never recall mode
+  bm25?: { docs: Doc[]; trigrams?: Map<string, Set<string>> };
+  fileComplexity?: Map<string, number>; // rel → whole-file branch count + 1 (code files)
+}
+
+const caches = new WeakMap<RepoScan, DerivedCache>();
+
+function cacheFor(scan: RepoScan): DerivedCache {
+  let c = caches.get(scan);
+  if (!c) caches.set(scan, (c = {}));
+  return c;
+}
+
+export function resolveContextFor(scan: RepoScan): ResolveContext {
+  const c = cacheFor(scan);
+  return (c.resolveCtx ??= buildResolveContext(scan));
+}
+
+// The same pair set computeImportPairs (callers.ts) historically built —
+// computed here against the memoized resolve context so the public function
+// can delegate (returning a fresh Set) without a recompute per call.
+export function importPairsFor(scan: RepoScan): Set<string> {
+  const c = cacheFor(scan);
+  if (!c.importPairs) {
+    const ctx = resolveContextFor(scan);
+    const pairs = new Set<string>();
+    for (const f of scan.files) {
+      for (const ref of f.refs) {
+        if (ref.kind !== "import") continue;
+        const r = resolveImport(f.rel, f.ext, ref.spec, ctx);
+        if (r.kind === "resolved" && r.target !== f.rel) pairs.add(`${f.rel}|${r.target}`);
+      }
+    }
+    c.importPairs = pairs;
+  }
+  return c.importPairs;
+}
+
+export function uniqueDefsFor(scan: RepoScan): Map<string, string> {
+  const c = cacheFor(scan);
+  return (c.uniqueDefs ??= uniqueSymbolDefs(scan));
+}
+
+export function symbolRefsFor(scan: RepoScan): Map<string, Set<string>> {
+  const c = cacheFor(scan);
+  return (c.symbolRefs ??= computeSymbolRefs(scan));
+}
+
+// Default (precision) caller index only. Recall-mode indexes are option-
+// dependent and stay unmemoized on buildCallerIndex's public path.
+export function callerIndexFor(scan: RepoScan): CallerIndex {
+  const c = cacheFor(scan);
+  return (c.callerIndex ??= buildCallerIndex(scan, importPairsFor(scan)));
+}
+
+export function bm25DocsFor(scan: RepoScan): Doc[] {
+  const c = cacheFor(scan);
+  return (c.bm25 ??= { docs: buildDocs(scan) }).docs;
+}
+
+// The corpus-vocabulary trigram index stays LAZY: built only when searchIndex
+// first meets a zero-df query term (the pre-cache behavior), then cached so
+// later fuzzy queries on the same scan skip the rebuild.
+export function bm25TrigramsFor(scan: RepoScan): Map<string, Set<string>> {
+  const c = cacheFor(scan);
+  const bm25 = (c.bm25 ??= { docs: buildDocs(scan) });
+  return (bm25.trigrams ??= buildTrigramIndex(bm25.docs));
+}
+
+// Whole-file branch counts for every code file (riskHotspots' per-file
+// complexity). The FIRST call still reads each code file from disk — accepted;
+// repeat calls on the same scan become lookups.
+export function fileComplexityFor(scan: RepoScan): Map<string, number> {
+  const c = cacheFor(scan);
+  if (!c.fileComplexity) {
+    const m = new Map<string, number>();
+    for (const f of scan.files) {
+      if (f.kind !== "code") continue;
+      m.set(f.rel, complexityOfSource(readText(join(scan.root, f.rel))));
+    }
+    c.fileComplexity = m;
+  }
+  return c.fileComplexity;
+}

package/src/embed/model.ts

@@ -87,20 +87,17 @@ export function hasEmbedModel(repo?: string): boolean {
   return resolveEmbedModelDir(repo) !== undefined;
 }
 
-// Load and validate a static model from a directory containing model.json.
-// Throws on a malformed file (bad dim, ragged weights) so a corrupt asset fails
-// loudly at load rather than silently misranking. Returns undefined only when
-// the directory has no model.json (the not-present case).
-export function loadEmbedModel(dir?: string): StaticEmbedModel | undefined {
-  if (!dir) return undefined;
-  const path = join(dir, "model.json");
-  if (!existsSync(path)) return undefined;
-  const raw = JSON.parse(readFileSync(path, "utf8")) as ModelFile;
-  const { modelId, dim, vocab, weights } = raw;
-  if (typeof modelId !== "string" || !modelId) throw new Error(`embed model: missing modelId in ${path}`);
-  if (!Number.isInteger(dim) || dim <= 0) throw new Error(`embed model: bad dim ${dim} in ${path}`);
+// Validate a parsed model.json body and construct the loaded model. Throws on
+// a malformed shape (missing modelId, bad dim, vocab/weights mismatch, ragged
+// rows) with a message citing `source` (a file path, a URL, …) so a corrupt
+// asset fails loudly wherever it enters — at load, or at `embed pull` BEFORE
+// the shape-invalid file is ever written to disk.
+export function parseEmbedModel(raw: unknown, source: string): StaticEmbedModel {
+  const { modelId, dim, vocab, weights, unk: rawUnk } = (raw ?? {}) as ModelFile;
+  if (typeof modelId !== "string" || !modelId) throw new Error(`embed model: missing modelId in ${source}`);
+  if (!Number.isInteger(dim) || dim <= 0) throw new Error(`embed model: bad dim ${dim} in ${source}`);
   if (!Array.isArray(vocab) || !Array.isArray(weights) || vocab.length !== weights.length) {
-    throw new Error(`embed model: vocab/weights length mismatch in ${path}`);
+    throw new Error(`embed model: vocab/weights length mismatch in ${source}`);
   }
   const vocabSize = vocab.length;
   const flat = new Float64Array(vocabSize * dim);
@@ -115,11 +112,23 @@ export function loadEmbedModel(dir?: string): StaticEmbedModel | undefined {
     }
     for (let d = 0; d < dim; d++) flat[i * dim + d] = Number(row[d]);
   }
-  const unk = typeof raw.unk === "string" ? raw.unk : "[UNK]";
+  const unk = typeof rawUnk === "string" ? rawUnk : "[UNK]";
   const unkId = vmap.has(unk) ? vmap.get(unk)! : -1;
   return { modelId, dim, unk, unkId, vocabSize, vocab: vmap, weights: flat };
 }
 
+// Load and validate a static model from a directory containing model.json.
+// Throws on a malformed file (bad dim, ragged weights) so a corrupt asset fails
+// loudly at load rather than silently misranking. Returns undefined only when
+// the directory has no model.json (the not-present case).
+export function loadEmbedModel(dir?: string): StaticEmbedModel | undefined {
+  if (!dir) return undefined;
+  const path = join(dir, "model.json");
+  if (!existsSync(path)) return undefined;
+  const raw = JSON.parse(readFileSync(path, "utf8")) as unknown;
+  return parseEmbedModel(raw, path);
+}
+
 // What `embed pull` fetches, and whether to verify it. CODEINDEX_EMBED_URL wins
 // outright (the user's explicit override / private mirror) and carries NO
 // sha256, so a custom asset keeps the current un-verified behavior. With no env,