@maxgfr/codeindex 2.17.1 → 2.19.0

package/src/pool.ts

@@ -0,0 +1,271 @@
+// Parallel extraction across worker_threads.
+//
+// Cold indexing is ~90% per-file extraction, and extractCode is a pure function
+// of (rel, ext, content) — so it parallelises exactly. Measured on
+// facebook/react (4343 JS/TS files, 10-core M5): 4.04s sequential -> 0.94s on
+// eight workers, with peak RSS BELOW the single-threaded full index because
+// each worker's tree-sitter wasm arena is torn down when it exits (nothing
+// inside one process ever returns that memory).
+//
+// Determinism is not weakened. Records come back out of order and are keyed by
+// path, and scanRepo assembles + sorts exactly as it always did; every record is
+// built by the SAME buildCodeRecord the sequential path uses. The one way a
+// worker could produce a different record is by having a different grammar tier
+// than the main thread — so workers report the grammars they actually loaded and
+// any disagreement aborts the whole parallel run back to sequential.
+//
+// Sequential fallback is always available and always correct: any failure to
+// resolve, spawn, or agree returns undefined and the caller scans as before.
+import { existsSync, statSync } from "node:fs";
+import { availableParallelism } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { Worker } from "node:worker_threads";
+import type { FileRecord } from "./types.js";
+import { sha1 } from "./hash.js";
+import { readText, walk } from "./walk.js";
+import { extToLang } from "./lang/registry.js";
+import { ensureGrammars, grammarKeysForExts, grammarReady } from "./ast/loader.js";
+import { buildCodeRecord, keptCodeFiles, scanRepo, type ExtractedRecord, type RepoScan, type ScanOptions } from "./scan.js";
+
+// One unit of work: a code file to read, hash and extract.
+interface Job {
+  abs: string;
+  rel: string;
+  ext: string;
+}
+
+interface WorkerInput {
+  jobs: Job[];
+  grammarKeys: string[];
+  maxCallsPerFile?: number;
+}
+
+interface WorkerOutput {
+  // The grammar keys this worker actually got ready. Compared against the main
+  // thread's — a mismatch means this worker would have silently used the regex
+  // tier for some language, so the run is discarded.
+  ready: string[];
+  records: { rel: string; size: number; mtimeMs: number; record: FileRecord }[];
+}
+
+// The URL a worker should import to reach THIS engine.
+//
+// Resolved the way grammars are (src/ast/loader.ts): look next to the running
+// module. When the bundle is the file itself (`scripts/engine.mjs`, whether from
+// npm or vendored) that is the module; when a consumer has re-bundled the engine
+// into their own entry, `engine.mjs` is not adjacent and we return undefined so
+// the caller stays sequential — importing THEIR bundle in a worker would run
+// their top-level code, which is exactly the hazard src/engine.ts:215 warns
+// about.
+function resolveEngineUrl(): string | undefined {
+  try {
+    const here = fileURLToPath(import.meta.url);
+    if (here.endsWith("engine.mjs")) return pathToFileURL(here).href;
+    const adjacent = join(dirname(here), "engine.mjs");
+    if (existsSync(adjacent)) return pathToFileURL(adjacent).href;
+    return undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+// Deadlock guard for a worker that never answers. Generous on purpose: a shard
+// of a very large repo can legitimately take minutes, and tripping this only
+// costs a fallback to the sequential scan.
+const WORKER_TIMEOUT_MS = 10 * 60 * 1000;
+
+// How many workers to run. `CODEINDEX_WORKERS` wins when set; 0 or 1 means
+// sequential. Default leaves a core for the main thread and caps at 8 — past
+// that the wasm arena per worker costs more than the wall-clock it saves.
+export function workerCount(requested?: number): number {
+  const env = process.env["CODEINDEX_WORKERS"];
+  const raw = requested ?? (env !== undefined && env !== "" ? Number(env) : undefined);
+  if (raw !== undefined) return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 0;
+  let cores = 1;
+  try {
+    cores = availableParallelism();
+  } catch {
+    cores = 1;
+  }
+  return Math.max(0, Math.min(cores - 1, 8));
+}
+
+// ---------------------------------------------------------------------------
+// Worker side
+// ---------------------------------------------------------------------------
+
+// Runs inside a worker: load the grammars, then read/hash/extract each job.
+// Exported (and named) so the bootstrap below can find it by name — that lookup
+// failing is itself the signal that we imported something that is not this
+// engine, and the worker exits without touching the caller's data.
+export async function runExtractWorker(input: WorkerInput, post: (out: WorkerOutput) => void): Promise<void> {
+  await ensureGrammars(input.grammarKeys);
+  const ready = input.grammarKeys.filter((k) => grammarReady(k));
+  const records: WorkerOutput["records"] = [];
+  for (const job of input.jobs) {
+    let size: number;
+    let mtimeMs: number;
+    try {
+      const st = statSync(job.abs);
+      size = st.size;
+      mtimeMs = st.mtimeMs;
+    } catch {
+      continue; // vanished mid-run — the main thread re-reads it (and drops it)
+    }
+    const content = readText(job.abs);
+    const record = buildCodeRecord(job.rel, job.ext, size, content, sha1(content), extToLang(job.ext), {
+      maxCallsPerFile: input.maxCallsPerFile,
+    });
+    records.push({ rel: job.rel, size, mtimeMs, record });
+  }
+  post({ ready, records });
+}
+
+// ---------------------------------------------------------------------------
+// Main side
+// ---------------------------------------------------------------------------
+
+// Extract `jobs` across `count` workers. Returns undefined — meaning "scan
+// sequentially" — when workers cannot be used or cannot be trusted:
+//   * the engine URL does not resolve to an on-disk engine.mjs,
+//   * a worker fails to spawn, errors, or reports an error payload,
+//   * a worker's ready grammar set differs from the main thread's.
+// Never throws.
+export async function extractInParallel(
+  jobs: Job[],
+  grammarKeys: string[],
+  count: number,
+  opts: { maxCallsPerFile?: number } = {},
+): Promise<Map<string, ExtractedRecord> | undefined> {
+  if (count < 2 || jobs.length === 0) return undefined;
+  const engineUrl = resolveEngineUrl();
+  if (!engineUrl) return undefined;
+
+  // Ask workers for exactly the grammars the MAIN THREAD has ready — not for
+  // every grammar the repo's extensions imply.
+  //
+  // This is what makes the tiers agree by construction. Under `--no-ast` (or any
+  // caller that never warmed grammars) the main thread would extract with regex,
+  // so the workers must too; asking for the walk-derived set instead would have
+  // them load wasm, produce AST-tier records, mismatch, and silently drop the
+  // whole run back to sequential — losing parallelism on exactly the tier where
+  // it is cheapest.
+  const wanted = grammarKeys.filter((k) => grammarReady(k)).sort();
+
+  // Round-robin over the path-sorted job list, so every shard sees the same
+  // language mix and no shard loads a grammar the others don't.
+  const shards: Job[][] = Array.from({ length: Math.min(count, jobs.length) }, () => []);
+  jobs.forEach((j, i) => shards[i % shards.length]!.push(j));
+
+  const bootstrap =
+    `import { runExtractWorker } from ${JSON.stringify(engineUrl)};\n` +
+    `import { parentPort, workerData } from "node:worker_threads";\n` +
+    `runExtractWorker(workerData.input, (o) => parentPort.postMessage(o))` +
+    `.catch((e) => parentPort.postMessage({ error: String(e) }));\n`;
+
+  try {
+    const outputs = await Promise.all(
+      shards.map(
+        (jobsForShard) =>
+          new Promise<WorkerOutput | { error: string }>((resolve, reject) => {
+            const w = new Worker(bootstrap, {
+              eval: true,
+              workerData: { input: { jobs: jobsForShard, grammarKeys: wanted, maxCallsPerFile: opts.maxCallsPerFile } },
+            });
+            // A worker that neither answers nor errors would hang the whole
+            // index, since Promise.all waits forever. The bound is a deadlock
+            // guard, not a performance knob — tripping it costs a fallback to
+            // the sequential scan, which is always correct.
+            const timer = setTimeout(() => {
+              reject(new Error("extraction worker timed out"));
+              void w.terminate();
+            }, WORKER_TIMEOUT_MS);
+            const settle = (fn: () => void): void => {
+              clearTimeout(timer);
+              fn();
+            };
+            w.once("message", (m: WorkerOutput | { error: string }) => {
+              settle(() => resolve(m));
+              void w.terminate();
+            });
+            w.once("error", (e) => settle(() => reject(e)));
+            w.once("exit", (code) => {
+              // Already-settled rejects are no-ops; terminate() itself exits non-zero.
+              if (code !== 0) settle(() => reject(new Error(`extraction worker exited with ${code}`)));
+            });
+          }),
+      ),
+    );
+
+    const out = new Map<string, ExtractedRecord>();
+    for (const o of outputs) {
+      if ("error" in o) return undefined;
+      // A worker that readied a different grammar set would have produced
+      // regex-tier records for some language — discard the whole run rather
+      // than emit a mix of tiers.
+      if (o.ready.slice().sort().join(",") !== wanted.join(",")) return undefined;
+      for (const r of o.records) out.set(r.rel, { size: r.size, mtimeMs: r.mtimeMs, record: r.record });
+    }
+    return out;
+  } catch {
+    return undefined;
+  }
+}
+
+// scanRepo, with the code files extracted across workers.
+//
+// The parallel part is strictly a pre-pass: it produces the SAME records
+// scanRepo would have built, hands them over via ScanOptions.extracted, and
+// scanRepo then runs its normal loop — cache fastpaths, docs, ordering and
+// change-tracking all unchanged. Anything that prevents or invalidates the
+// parallel pass simply leaves `extracted` empty, and the result is exactly a
+// sequential scan.
+//
+// `opts.workers` overrides the worker count (0/1 = sequential); absent, see
+// workerCount() and CODEINDEX_WORKERS.
+export async function scanRepoParallel(
+  root: string,
+  opts: ScanOptions & { workers?: number } = {},
+): Promise<RepoScan> {
+  const count = workerCount(opts.workers);
+  if (count < 2) return scanRepo(root, opts);
+
+  // Walk ONCE and reuse it for both the job list and the scan itself.
+  const walked =
+    opts.precomputedWalk ??
+    walk(root, {
+      maxFileBytes: opts.maxBytes,
+      maxFiles: opts.maxFiles,
+      gitignore: opts.gitignore,
+      ignoreDirs: opts.ignoreDirs,
+    });
+  const scanOpts: ScanOptions = { ...opts, precomputedWalk: walked };
+
+  // Only code files are worth shipping out: docs must be read on the main
+  // thread anyway (the graph's mention pass needs their text), and everything
+  // else is a read plus a hash with no extraction behind it.
+  //
+  // Files the cache will serve by its stat fastpath are skipped — extracting
+  // them would be work whose result scanRepo discards.
+  const jobs: Job[] = [];
+  for (const { f } of keptCodeFiles(root, scanOpts)) {
+    const cached = opts.cache?.get(f.rel);
+    if (
+      !opts.fullHash &&
+      cached &&
+      cached.size !== undefined &&
+      cached.mtimeMs !== undefined &&
+      cached.size === f.size &&
+      cached.mtimeMs === f.mtimeMs
+    ) {
+      continue;
+    }
+    jobs.push({ abs: f.abs, rel: f.rel, ext: f.ext });
+  }
+  if (jobs.length === 0) return scanRepo(root, scanOpts);
+
+  const grammarKeys = grammarKeysForExts(walked.files.map((f) => f.ext));
+  const extracted = await extractInParallel(jobs, grammarKeys, count, { maxCallsPerFile: opts.maxCallsPerFile });
+  return scanRepo(root, extracted ? { ...scanOpts, extracted } : scanOpts);
+}

package/src/preload.ts

@@ -0,0 +1,159 @@
+// Reusing a persisted `.codeindex/` index instead of rebuilding it.
+//
+// `codeindex index` writes graph.json + symbols.json + cache.json. This module
+// reads them back, and it does so in two independent steps:
+//
+//   * cache.json seeds a scan, so every unchanged file takes scan.ts's stat
+//     fastpath instead of a read + hash + extraction pass;
+//   * when the freshness guard holds, graph.json/symbols.json are deserialized
+//     straight in, so the whole downstream pipeline is skipped too.
+//
+// Both are pure optimizations. Seeded records come from scan.ts's own reuse
+// paths, so they are value-identical to a cold scan's, and the guard is the
+// SAME oracle the CLI's index fastpath uses to prove the on-disk artifacts equal
+// a fresh build. Anything absent, stale, corrupt, or mismatched falls back to
+// the cold path exactly — never a throw.
+//
+// This started life inside the MCP server, which was the only caller. The CLI's
+// read commands (search, symbols, callers, graph, repomap, hotspots, deadcode,
+// complexity, mermaid, rules) rebuilt from scratch on every invocation, so
+// `codeindex search` cost a full tree-sitter pass over the repo every time it
+// ran — 6.3s on a 7k-file repo with a fresh index sitting right next to it.
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { ENGINE_VERSION, EXTRACTOR_VERSION, SCHEMA_VERSION } from "./types.js";
+import type { FileRecord, Graph, SymbolIndex } from "./types.js";
+import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js";
+import type { IndexArtifacts } from "./pipeline.js";
+import { sha1 } from "./hash.js";
+
+// The default index location, relative to the repo root.
+export const INDEX_DIR = ".codeindex";
+
+export type PersistedCacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number };
+export type PersistedCacheMap = Map<string, PersistedCacheEntry>;
+
+// ADDITIVE cache.json meta describing the artifacts a prior `index` run wrote
+// (see engine-cli.ts's CacheMeta). Old caches lacking these keys simply never
+// pass the guard below — their per-file records are still reused to seed the
+// scan. Only the graph/symbols shas matter here; the embed sidecar has its own
+// memoization path.
+export interface PersistedMeta {
+  engineVersion?: string;
+  commit?: string;
+  graphSha1?: string;
+  symbolsSha1?: string;
+}
+
+// A scan re-expressed as the `ScanOptions.cache` shape (the exact map the CLI
+// persists as cache.json): rel → (hash, record, size, mtimeMs), so the next
+// scanRepo can take the stat fastpath / hash-match reuse paths against it.
+export function toCacheMap(scan: RepoScan): PersistedCacheMap {
+  const m: PersistedCacheMap = new Map();
+  for (const f of scan.files) m.set(f.rel, { hash: f.hash, record: f, size: f.size, mtimeMs: scan.mtimes.get(f.rel) });
+  return m;
+}
+
+// Read <indexDir>/cache.json into the (cacheMap, meta) the preload needs.
+// Per-file records are reusable ONLY when (schemaVersion, extractorVersion)
+// match this engine — the exact gate the CLI applies before trusting a cache —
+// otherwise the whole cache is discarded (cold scan). Any read/parse failure (no
+// index yet, unreadable, malformed) returns undefined: the cold path.
+export function readPersistedIndex(
+  repo: string,
+  indexDir: string = INDEX_DIR,
+): { cacheMap: PersistedCacheMap; meta: PersistedMeta } | undefined {
+  let parsed:
+    | ({ schemaVersion?: number; extractorVersion?: number; files?: Record<string, PersistedCacheEntry> } & PersistedMeta)
+    | undefined;
+  try {
+    parsed = JSON.parse(readFileSync(join(repo, indexDir, "cache.json"), "utf8")) as typeof parsed;
+  } catch {
+    return undefined;
+  }
+  if (!parsed || parsed.schemaVersion !== SCHEMA_VERSION || parsed.extractorVersion !== EXTRACTOR_VERSION || !parsed.files) {
+    return undefined;
+  }
+  return {
+    cacheMap: new Map(Object.entries(parsed.files)),
+    meta: {
+      engineVersion: parsed.engineVersion,
+      commit: parsed.commit,
+      graphSha1: parsed.graphSha1,
+      symbolsSha1: parsed.symbolsSha1,
+    },
+  };
+}
+
+// The freshness guard, applied to a scan seeded from cache.json:
+// contentUnchanged proves this scan's records are the ones that built the
+// on-disk artifacts; engineVersion pins the version stamp graph.json embeds and
+// commit the HEAD it embeds; the sha checks prove the on-disk bytes ARE that
+// build's output. All true ⇒ graph.json/symbols.json are byte-equal to
+// buildArtifactsFromScan(scan) run here, so deserialize them instead of
+// rebuilding. Graph/SymbolIndex are pure JSON POJOs (no Map/Set/typed fields),
+// so JSON.parse is a lossless round-trip — a schemaVersion assert is the only
+// reconstruction needed. ANY failure — a stale scan, a version/commit/sha
+// mismatch, a missing/corrupt/partial artifact, an unexpected schemaVersion —
+// returns undefined so the caller rebuilds. NEVER throws (a corrupt artifact
+// must degrade, not crash the caller).
+export function preloadArtifacts(
+  repo: string,
+  scan: RepoScan,
+  meta: PersistedMeta,
+  indexDir: string = INDEX_DIR,
+): IndexArtifacts | undefined {
+  if (
+    !scan.contentUnchanged ||
+    meta.engineVersion !== ENGINE_VERSION ||
+    meta.commit !== scan.commit ||
+    meta.graphSha1 === undefined ||
+    meta.symbolsSha1 === undefined
+  ) {
+    return undefined;
+  }
+  const dir = join(repo, indexDir);
+  let graphBytes: Buffer;
+  let symbolsBytes: Buffer;
+  try {
+    graphBytes = readFileSync(join(dir, "graph.json"));
+    symbolsBytes = readFileSync(join(dir, "symbols.json"));
+  } catch {
+    return undefined; // a sha'd artifact went missing since cache.json — rebuild
+  }
+  // sha over the raw bytes; sha1(string) hashes the same UTF-8 bytes writeFileSync
+  // put on disk, so this equals the meta sha the CLI computed over the render.
+  if (sha1(graphBytes) !== meta.graphSha1 || sha1(symbolsBytes) !== meta.symbolsSha1) {
+    return undefined; // tampered / partial / corrupt on-disk bytes — rebuild
+  }
+  try {
+    const graph = JSON.parse(graphBytes.toString("utf8")) as Graph;
+    const symbols = JSON.parse(symbolsBytes.toString("utf8")) as SymbolIndex;
+    if (graph.schemaVersion !== SCHEMA_VERSION || symbols.schemaVersion !== SCHEMA_VERSION) return undefined;
+    return { scan, graph, symbols };
+  } catch {
+    // Unreachable once the shas matched (the bytes are valid JSON this engine
+    // wrote), but the contract is "never throw" — degrade to a rebuild.
+    return undefined;
+  }
+}
+
+// Seed a scan from cache.json and, when the guard holds, the artifacts from
+// graph.json/symbols.json. undefined ⇒ no usable persisted index ⇒ the caller
+// takes the cold path unchanged.
+export function preloadSession(
+  repo: string,
+  opts: Omit<ScanOptions, "cache">,
+  indexDir: string = INDEX_DIR,
+): { scan: RepoScan; cacheMap: PersistedCacheMap; arts?: IndexArtifacts } | undefined {
+  const persisted = readPersistedIndex(repo, indexDir);
+  if (!persisted) return undefined;
+  // Seed the scan from the persisted records — scan.ts's stat fastpath + exact
+  // content-hash reuse make this value-identical to a cold scan, only cheaper,
+  // and it computes the contentUnchanged the artifact guard reads. When the
+  // on-disk content drifted from cache.json, changed files are re-read/extracted
+  // here exactly as a cold scan would, so the scan stays correct and the guard
+  // simply fails (arts undefined → rebuild on demand).
+  const scan = scanRepo(repo, { ...opts, cache: persisted.cacheMap });
+  return { scan, cacheMap: toCacheMap(scan), arts: preloadArtifacts(repo, scan, persisted.meta, indexDir) };
+}

package/src/render/scip.ts

@@ -31,12 +31,64 @@ export interface RenderScipOptions {
 }
 
 // ---------------------------------------------------------------------------
-// Protobuf wire-format primitives (proto3). Messages are assembled as plain
-// number[] byte buffers and embedded into their parent as length-delimited
-// (wire type 2). Only the two wire types we use are implemented: varint (0)
-// and length-delimited (2).
+// Protobuf wire-format primitives (proto3). Messages are assembled into byte
+// sinks and embedded into their parent as length-delimited (wire type 2). Only
+// the two wire types we use are implemented: varint (0) and length-delimited (2).
 // ---------------------------------------------------------------------------
-type Bytes = number[];
+
+// A growable byte buffer. This used to be a plain `number[]` — one boxed JS
+// number per OUTPUT BYTE, which on a 7k-file repo cost ~176 MB of RSS to emit
+// an 8 MB index. A doubling Uint8Array holds the same bytes at 1 byte each.
+class Bytes {
+  private buf: Uint8Array;
+  private len = 0;
+
+  constructor(capacity = 64) {
+    this.buf = new Uint8Array(capacity);
+  }
+
+  get length(): number {
+    return this.len;
+  }
+
+  private grow(need: number): void {
+    if (this.len + need <= this.buf.length) return;
+    let cap = this.buf.length * 2 || 64;
+    while (cap < this.len + need) cap *= 2;
+    const next = new Uint8Array(cap);
+    next.set(this.buf.subarray(0, this.len));
+    this.buf = next;
+  }
+
+  // Rewind without releasing the buffer, so a per-item scratch sink is
+  // allocated once per document instead of once per occurrence/symbol.
+  reset(): void {
+    this.len = 0;
+  }
+
+  push(byte: number): void {
+    this.grow(1);
+    this.buf[this.len++] = byte;
+  }
+
+  pushAll(src: ArrayLike<number>): void {
+    this.grow(src.length);
+    if (src instanceof Uint8Array) this.buf.set(src, this.len);
+    else for (let i = 0; i < src.length; i++) this.buf[this.len + i] = src[i]!;
+    this.len += src.length;
+  }
+
+  // A view over exactly the bytes written — no copy. Callers must not retain it
+  // across further writes to this sink.
+  view(): Uint8Array {
+    return this.buf.subarray(0, this.len);
+  }
+
+  toUint8Array(): Uint8Array {
+    return this.buf.slice(0, this.len);
+  }
+}
+
 const utf8 = new TextEncoder();
 
 // Unsigned LEB128. Every value we encode (field tags, enum values, ranges,
@@ -62,7 +114,11 @@ function pushVarintField(out: Bytes, field: number, n: number): void {
 function pushLenDelim(out: Bytes, field: number, payload: ArrayLike<number>): void {
   pushTag(out, field, 2);
   pushVarint(out, payload.length);
-  for (let i = 0; i < payload.length; i++) out.push(payload[i]!);
+  out.pushAll(payload);
+}
+
+function pushMessage(out: Bytes, field: number, payload: Bytes): void {
+  pushLenDelim(out, field, payload.view());
 }
 
 function pushString(out: Bytes, field: number, s: string): void {
@@ -73,9 +129,9 @@ function pushString(out: Bytes, field: number, s: string): void {
 // the concatenated varints of every element (this is the piece the plain
 // per-element encoding would get wrong).
 function pushPackedInt32(out: Bytes, field: number, values: number[]): void {
-  const payload: Bytes = [];
+  const payload = new Bytes(values.length * 2);
   for (const v of values) pushVarint(payload, v);
-  pushLenDelim(out, field, payload);
+  pushMessage(out, field, payload);
 }
 
 // ---------------------------------------------------------------------------
@@ -330,25 +386,27 @@ export function renderScip(scan: RepoScan, opts: RenderScipOptions = {}): Uint8A
       }))
       .sort((a, b) => byStr(a.symbol, b.symbol));
 
-    const doc: Bytes = [];
+    const doc = new Bytes(1024);
     pushString(doc, F_DOC_RELPATH, f.rel);
+    const ob = new Bytes(64);
     for (const o of occs) {
       const key = `${o.range.join(",")} ${o.roles} ${o.symbol}`;
       if (seenOcc.has(key)) continue;
       seenOcc.add(key);
-      const ob: Bytes = [];
+      ob.reset();
       pushPackedInt32(ob, F_OCC_RANGE, o.range);
       pushString(ob, F_OCC_SYMBOL, o.symbol);
       if (o.roles !== 0) pushVarintField(ob, F_OCC_ROLES, o.roles);
-      pushLenDelim(doc, F_DOC_OCCURRENCES, ob);
+      pushMessage(doc, F_DOC_OCCURRENCES, ob);
     }
+    const sb = new Bytes(64);
     for (const si of infos) {
-      const sb: Bytes = [];
+      sb.reset();
       pushString(sb, F_SI_SYMBOL, si.symbol);
       if (si.kind !== undefined) pushVarintField(sb, F_SI_KIND, si.kind);
       pushString(sb, F_SI_DISPLAY_NAME, si.displayName);
       if (si.enclosing) pushString(sb, F_SI_ENCLOSING, si.enclosing);
-      pushLenDelim(doc, F_DOC_SYMBOLS, sb);
+      pushMessage(doc, F_DOC_SYMBOLS, sb);
     }
     pushString(doc, F_DOC_LANGUAGE, f.lang);
     pushVarintField(doc, F_DOC_POSITION_ENCODING, POSITION_ENCODING_UTF16);
@@ -356,19 +414,21 @@ export function renderScip(scan: RepoScan, opts: RenderScipOptions = {}): Uint8A
   }
 
   // Metadata { tool_info, project_root, text_document_encoding }.
-  const toolInfo: Bytes = [];
+  const toolInfo = new Bytes();
   pushString(toolInfo, F_TOOL_NAME, "codeindex");
   pushString(toolInfo, F_TOOL_VERSION, toolVersion);
 
-  const metadata: Bytes = [];
-  pushLenDelim(metadata, F_META_TOOL_INFO, toolInfo);
+  const metadata = new Bytes();
+  pushMessage(metadata, F_META_TOOL_INFO, toolInfo);
   pushString(metadata, F_META_PROJECT_ROOT, projectRoot);
   pushVarintField(metadata, F_META_TEXT_ENCODING, TEXT_ENCODING_UTF8);
 
   // Index { metadata, documents }.
-  const index: Bytes = [];
-  pushLenDelim(index, F_INDEX_METADATA, metadata);
-  for (const d of documents) pushLenDelim(index, F_INDEX_DOCUMENTS, d);
+  let total = 0;
+  for (const d of documents) total += d.length;
+  const index = new Bytes(total + metadata.length + 16);
+  pushMessage(index, F_INDEX_METADATA, metadata);
+  for (const d of documents) pushMessage(index, F_INDEX_DOCUMENTS, d);
 
-  return Uint8Array.from(index);
+  return index.toUint8Array();
 }

package/src/render/symbols-json.ts

@@ -2,14 +2,15 @@ import { SCHEMA_VERSION } from "../types.js";
 import type { SymbolIndex } from "../types.js";
 import type { RepoScan } from "../scan.js";
 import { byStr } from "../sort.js";
-import { uniqueSymbolDefs } from "../graph.js";
+import { uniqueDefsFor } from "../derived.js";
 
 // Files that REFERENCE each unique exported symbol — code files via their AST
 // identifiers, doc files via naming the symbol. Feeds symbols.json `refs` so
 // `symbols <name>` can answer "where is X used?". Mirrors the graph's use/mention
 // eligibility (unique + distinctive), so refs and edges stay consistent.
 export function computeSymbolRefs(scan: RepoScan): Map<string, Set<string>> {
-  const unique = uniqueSymbolDefs(scan);
+  // Memoized: buildGraph built exactly this map moments earlier in the pipeline.
+  const unique = uniqueDefsFor(scan);
   const refs = new Map<string, Set<string>>();
   if (!unique.size) return refs;
   const add = (name: string, file: string): void => {