@herbertgao/pi-extensions 2026.8.13 → 2026.8.15

package/node_modules/pi-lens/dist/clients/word-index.js

@@ -16,11 +16,14 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { createDeadline, forEachCooperatively, yieldIfOverBudget, } from "./cooperative-budget.js";
+import { incrementDegradationCount } from "./degradation-ledger.js";
 import { KIND_EXTENSIONS } from "./file-kinds.js";
 import { PathKeyedMap } from "./path-keyed-map.js";
 import { isAtOrAboveHomeDir, normalizeEphemeralMapKey } from "./path-utils.js";
+import { createSingleFlight } from "./single-flight.js";
 import { createDebounceScheduler, } from "./persist-debounce.js";
 import { getWordIndexMaxFilesDerived } from "./project-scale.js";
+import { compactPostingsIntoArena, compactPostingsIntoArenaCooperatively, countPostingBackingStores, countPostingEntries, estimateWordIndexStoreBytes, WordForwardEntry, WordIndexFileTable, WordPostingList, } from "./word-index-store.js";
 import { logWordIndex } from "./word-index-logger.js";
 /**
  * The single normalizer every word-index path key folds through (#1025). The
@@ -160,6 +163,7 @@ const WORD_INDEX_LONG_LINE_YIELD_CHARS = 4096;
 function createEmptyWordIndex(truncated) {
     return {
         postings: new Map(),
+        fileTable: new WordIndexFileTable(),
         docLengths: new PathKeyedMap(wordIndexKey),
         totalTokens: 0,
         docCount: 0,
@@ -167,38 +171,274 @@ function createEmptyWordIndex(truncated) {
         forward: new PathKeyedMap(wordIndexKey),
         fileMtimes: new PathKeyedMap(wordIndexKey),
         fileSizes: new PathKeyedMap(wordIndexKey),
+        replacementStats: { count: 0, totalMs: 0, maxMs: 0 },
+        dirtyFiles: new Set(),
+        postingStoreCount: 0,
+        recompactFlight: createSingleFlight(),
     };
 }
-function indexWordLine(index, filePath, line, lineNumber, tokenLineCounts) {
+export function countWordIndexPostingEntries(index) {
+    return countPostingEntries(index);
+}
+/**
+ * Estimated resident bytes of the index's two packed stores (#2069).
+ * Reported on the `full_rebuild`, `incremental_refresh`, and `cold_build`
+ * word-index log records and on the `memory_sample` word-index subsystem, so
+ * a heap census can be reconciled against the log without taking a snapshot —
+ * the gap that let #1999 under-count this subsystem by a factor of sixty.
+ */
+export function estimateWordIndexResidentBytes(index) {
+    return estimateWordIndexStoreBytes(index);
+}
+/**
+ * Record a posting whose file id the file table cannot resolve (#2069).
+ *
+ * Unreachable by construction: {@link WordIndexFileTable.release} only runs
+ * after the forward index has enumerated and removed every posting naming that
+ * id. If it fires, that invariant broke, and the visible symptom is a query
+ * returning FEWER results with nothing to distinguish it from a smaller match
+ * set — the clean-versus-errored ambiguity AGENTS.md catalogs as shape 10.
+ *
+ * `incrementDegradationCount` rather than `recordDegradationOnce` because
+ * ranking is a hot loop: the tally stays exact for every occurrence while only
+ * the first per (kind, orphaned id) writes a durable row. Subject is the id
+ * itself, so aggregation keeps the discriminating identity.
+ */
+function recordOrphanWordIndexFileId(fileId, token, seam) {
+    incrementDegradationCount({
+        kind: "word-index-orphan-file-id",
+        subject: `fileId:${fileId}`,
+        reason: `${seam} dropped a posting for token "${token}": the file table has no path for this id`,
+    });
+}
+/**
+ * Decode one token’s postings into `{ file, line }` objects. Allocates, so it
+ * is for callers that genuinely need the display form (tests, diagnostics) —
+ * never for the hot ranking loop, which reads the packed lanes directly.
+ */
+export function wordIndexPostingHits(index, token) {
+    const list = index.postings.get(token);
+    if (!list)
+        return [];
+    const hits = [];
+    for (let i = 0; i < list.length; i += 1) {
+        const fileId = list.fileIdAt(i);
+        const file = index.fileTable.pathFor(fileId);
+        if (file === undefined) {
+            recordOrphanWordIndexFileId(fileId, token, "decode");
+            continue;
+        }
+        hits.push({ file, line: list.lineAt(i) });
+    }
+    return hits;
+}
+/**
+ * Settle a freshly built index's posting store into its compact resident form
+ * (#2069): re-home every list into one shared arena, sized from the live entry
+ * counts. That both releases the doubling growth slack a build leaves behind
+ * and stops the long tail of rare tokens paying for an `ArrayBuffer` header
+ * each.
+ */
+function compactWordIndexPostings(index) {
+    compactPostingsIntoArena(index.postings);
+    index.postingStoreCount = countPostingBackingStores(index.postings);
+}
+/** Bump the running distinct-backing-store tally by `count` (#2117). */
+function notePostingStoresAllocated(index, count) {
+    index.postingStoreCount = (index.postingStoreCount ?? 0) + count;
+}
+/**
+ * Floor for the recompaction gate: below this many backing stores the arena is
+ * never repacked, so a small index never pays an O(vocab) walk it cannot amortize
+ * (#2117). This is also the whole gate for indexes whose vocabulary is under
+ * `WORD_INDEX_RECOMPACT_STORE_FLOOR / WORD_INDEX_RECOMPACT_STORE_FRACTION`
+ * tokens, keeping the pre-#2246 behaviour for small corpora bit-identical.
+ */
+const WORD_INDEX_RECOMPACT_STORE_FLOOR = 64;
+/**
+ * Recompaction gate as a share of the live vocabulary (#2246). The flat 64-store
+ * threshold #2117 shipped was crossed by every single edit: one document
+ * replacement raises `postingStoreCount` by roughly the edited document's
+ * distinct-token count (hundreds), so the whole O(vocab) arena was rebuilt after
+ * every edit. Gating on a fraction of `postings.size` instead lets fragmentation
+ * accumulate proportionally to the index, so recompaction fires once per many
+ * edits rather than once per edit.
+ *
+ * Memory ceiling: the store count can reach a tenth of the vocabulary in private
+ * stores before a repack, each carrying a fixed header plus growth slack. #2117
+ * measured the fully-churned ceiling (every token private, store count =
+ * vocabulary) at +22% resident. The cost does not scale with the store count
+ * alone, because the tokens that churn most are also the ones carrying the
+ * longest posting lists. PR #2246's adversarial hot-token probe measured this
+ * gate at +13.7% resident at rest and +19.9% peak over base — inside the +22%
+ * #2117 accepted, but not by the wide margin a naive store-count ratio would
+ * predict. Raising the fraction further would cross that bound.
+ */
+const WORD_INDEX_RECOMPACT_STORE_FRACTION = 0.1;
+/**
+ * Backing-store count that triggers a repack for `index`: a fixed floor, or a
+ * share of the live vocabulary once the index is large enough for the share to
+ * exceed the floor. O(1) — reads `postings.size`, never walks the vocabulary.
+ */
+function wordIndexRecompactThreshold(index) {
+    return Math.max(WORD_INDEX_RECOMPACT_STORE_FLOOR, Math.floor(index.postings.size * WORD_INDEX_RECOMPACT_STORE_FRACTION));
+}
+/**
+ * Pack the arena if churn has spread the postings across more than the
+ * threshold of backing stores (#2117, #2246). Runs the exact O(vocab) store
+ * count as the authoritative gate — the per-edit hot path uses the O(1)
+ * `postingStoreCount` estimate, so this expensive walk only happens inside the
+ * bounded, off-hot-path recompaction it guards. Resets the estimate to the
+ * true post-compaction count.
+ */
+async function recompactWordIndexPostingsIfNeeded(index, root) {
+    const threshold = wordIndexRecompactThreshold(index);
+    const beforeStores = countPostingBackingStores(index.postings);
+    if (beforeStores <= threshold) {
+        index.postingStoreCount = beforeStores;
+        return;
+    }
+    const beforeBytes = estimateWordIndexResidentBytes(index);
+    await compactPostingsIntoArenaCooperatively(index.postings);
+    const afterStores = countPostingBackingStores(index.postings);
+    index.postingStoreCount = afterStores;
+    const afterBytes = estimateWordIndexResidentBytes(index);
+    const firstForRoot = incrementDegradationCount({
+        kind: "word-index-arena-recompact",
+        subject: path.resolve(root),
+        reason: `arena store threshold ${threshold} exceeded`,
+        metadata: { beforeBytes, afterBytes, beforeStores, afterStores },
+    });
+    // Keep the detailed record bounded per root. The ledger still counts every
+    // repeated recompaction and emits its own power-of-two summaries.
+    if (firstForRoot) {
+        logWordIndex({
+            phase: "incremental_refresh",
+            cwd: path.resolve(root),
+            trigger: "incremental_refresh",
+            reason: `arena_recompact beforeBytes=${beforeBytes} afterBytes=${afterBytes} beforeStores=${beforeStores} afterStores=${afterStores}`,
+        });
+    }
+}
+/**
+ * Serialize a recompaction through the per-index async operation queue so no
+ * async edit or refresh interleaves it (#2117). The queue plus the corruption-
+ * proof cooperative compactor together close the review-F2 data race: the queue
+ * excludes async operations, and the compactor's snapshot-and-skip publish
+ * excludes the synchronous per-edit path the queue cannot see. The per-index
+ * `recompactFlight` single-flight (#1753) keeps a burst of edits from stacking
+ * duplicates: a caller that arrives while one recompaction runs joins it rather
+ * than starting a second. The single slot uses a constant key.
+ */
+function enqueueWordIndexRecompact(index, root) {
+    if ((index.postingStoreCount ?? 0) <= wordIndexRecompactThreshold(index)) {
+        return Promise.resolve();
+    }
+    const flight = (index.recompactFlight ??= createSingleFlight());
+    return flight.run("recompact", () => enqueueAsyncWordIndexOperation(index, () => recompactWordIndexPostingsIfNeeded(index, root)));
+}
+/**
+ * O(1) hot-path gate for the synchronous per-edit seam. Reads the running
+ * store-count estimate, never a fresh `Set` over the vocabulary, then defers
+ * the actual work onto the async queue so it cannot stall the edit (#2117).
+ */
+function scheduleWordIndexRecompact(index, filePath) {
+    void enqueueWordIndexRecompact(index, path.dirname(filePath));
+}
+/**
+ * Test hook: settle any queued arena recompaction for `index`. Draining the
+ * index's own async operation queue is enough because the recompaction runs
+ * through it; there is no module-level scheduler to flush.
+ */
+export async function flushWordIndexRecompactionsForTests(index) {
+    await (asyncWordIndexOperations.get(index) ?? Promise.resolve());
+}
+function recordWordIndexReplacement(index, startedAt) {
+    const stats = index.replacementStats ?? { count: 0, totalMs: 0, maxMs: 0 };
+    const durationMs = Math.max(0, Date.now() - startedAt);
+    stats.count += 1;
+    stats.totalMs += durationMs;
+    stats.maxMs = Math.max(stats.maxMs, durationMs);
+    index.replacementStats = stats;
+}
+/** Intern a document path once per index; posting removal compares this id. */
+function internWordIndexFile(index, filePath) {
+    return index.fileTable.intern(wordIndexKey(filePath), filePath);
+}
+function appendWordIndexPostings(index, filePath, perTokenHits) {
+    const fileId = internWordIndexFile(index, filePath);
+    const tokenLineCounts = new Map();
+    for (const [token, lineNumbers] of perTokenHits) {
+        let list = index.postings.get(token);
+        if (list) {
+            // A grow allocates a fresh private store; count it so the O(1) gate
+            // tracks fragmentation without re-walking the vocabulary (#2117).
+            if (list.reserve(list.length + lineNumbers.length)) {
+                notePostingStoresAllocated(index, 1);
+            }
+        }
+        else {
+            list = new WordPostingList(token, lineNumbers.length);
+            index.postings.set(token, list);
+            notePostingStoresAllocated(index, 1);
+        }
+        // `list.token` is the canonical instance, so the forward entry points at
+        // the same string the postings map is keyed by instead of retaining this
+        // document’s own tokenizer allocation (#2069).
+        tokenLineCounts.set(list.token, lineNumbers.length);
+        for (const line of lineNumbers)
+            list.push(fileId, line);
+    }
+    return tokenLineCounts;
+}
+function commitWordIndexDocumentReplacement(index, doc, perTokenHits, docLength, startedAt) {
+    const tokenLineCounts = appendWordIndexPostings(index, doc.path, perTokenHits);
+    index.docLengths.set(doc.path, docLength);
+    index.forward.set(doc.path, WordForwardEntry.fromTally(tokenLineCounts));
+    index.fileMtimes.set(doc.path, -1);
+    index.fileSizes.set(doc.path, Buffer.byteLength(doc.content, "utf-8"));
+    index.dirtyFiles?.add(wordIndexKey(doc.path));
+    index.totalTokens += docLength;
+    index.docCount += 1;
+    recordWordIndexReplacement(index, startedAt);
+}
+function indexWordLine(index, fileId, line, lineNumber, tokenLineCounts) {
     const lineTokens = tokenizeLine(line);
     const seenOnLine = new Set();
     for (const token of lineTokens) {
         if (seenOnLine.has(token))
             continue;
         seenOnLine.add(token);
-        const arr = index.postings.get(token);
-        if (arr)
-            arr.push({ file: filePath, line: lineNumber });
-        else
-            index.postings.set(token, [{ file: filePath, line: lineNumber }]);
-        tokenLineCounts.set(token, (tokenLineCounts.get(token) ?? 0) + 1);
+        let list = index.postings.get(token);
+        if (!list) {
+            list = new WordPostingList(token, 1);
+            index.postings.set(token, list);
+        }
+        list.push(fileId, lineNumber);
+        // Canonical instance, not this line’s fresh tokenizer allocation (#2069).
+        const canonical = list.token;
+        tokenLineCounts.set(canonical, (tokenLineCounts.get(canonical) ?? 0) + 1);
     }
     return lineTokens.length;
 }
 function finishWordIndexDocument(index, doc, docLength, tokenLineCounts) {
     index.docLengths.set(doc.path, docLength);
-    index.forward?.set(doc.path, tokenLineCounts);
+    index.forward?.set(doc.path, WordForwardEntry.fromTally(tokenLineCounts));
     index.fileMtimes.set(doc.path, doc.mtimeMs ?? 0);
     index.fileSizes.set(doc.path, doc.size ?? Buffer.byteLength(doc.content, "utf-8"));
+    // A build has no prior wire cache. Keeping the marker makes a future
+    // serializer cache explicit and gives incremental refresh one uniform seam.
+    index.dirtyFiles?.add(wordIndexKey(doc.path));
     index.totalTokens += docLength;
     index.docCount += 1;
 }
 function indexWordDocument(index, doc) {
+    const fileId = internWordIndexFile(index, doc.path);
     const lines = doc.content.split(/\r?\n/);
     const tokenLineCounts = new Map();
     let docLength = 0;
     for (let i = 0; i < lines.length; i += 1) {
-        docLength += indexWordLine(index, doc.path, lines[i], i + 1, tokenLineCounts);
+        docLength += indexWordLine(index, fileId, lines[i], i + 1, tokenLineCounts);
     }
     finishWordIndexDocument(index, doc, docLength, tokenLineCounts);
 }
@@ -206,6 +446,7 @@ export function buildWordIndex(files) {
     const index = createEmptyWordIndex(files.truncated ?? false);
     for (const doc of files)
         indexWordDocument(index, doc);
+    compactWordIndexPostings(index);
     return index;
 }
 /**
@@ -221,12 +462,13 @@ export async function buildWordIndexAsync(files, shouldContinue = () => true) {
     for (const doc of files) {
         if (!shouldContinue())
             throw new Error("word index build superseded");
+        const fileId = internWordIndexFile(index, doc.path);
         const lines = doc.content.split(/\r?\n/);
         const tokenLineCounts = new Map();
         let docLength = 0;
         for (let i = 0; i < lines.length; i += 1) {
             const line = lines[i];
-            docLength += indexWordLine(index, doc.path, line, i + 1, tokenLineCounts);
+            docLength += indexWordLine(index, fileId, line, i + 1, tokenLineCounts);
             if (line.length >= WORD_INDEX_LONG_LINE_YIELD_CHARS ||
                 deadline.expired()) {
                 await yieldIfOverBudget(deadline);
@@ -240,6 +482,7 @@ export async function buildWordIndexAsync(files, shouldContinue = () => true) {
                 throw new Error("word index build superseded");
         }
     }
+    compactWordIndexPostings(index);
     return index;
 }
 /**
@@ -256,19 +499,27 @@ export function removeWordIndexDocument(index, filePath) {
     const tokenLineCounts = index.forward.get(filePath);
     if (!tokenLineCounts)
         return false;
-    // `postings` is token-keyed (not a PathKeyedMap), so its `WordHit.file`
-    // display strings must be compared through the SAME normalizer the path maps
-    // use — otherwise a build-form hit (`SUB/a.ts`) survives an edit-form removal
+    // `postings` is token-keyed (not a PathKeyedMap), so the file identity it
+    // carries must be resolved through the SAME normalizer the path maps use —
+    // otherwise a build-form hit (`SUB/a.ts`) survives an edit-form removal
     // (`sub/a.ts`) on a case-insensitive FS and lingers as a stale posting
-    // (the #1025 item #2 bug this fix closes).
+    // (the #1025 item #2 bug this fix closes). Since #2069 that identity is the
+    // file table’s integer id, so the comparison below is an int compare rather
+    // than a per-element string compare.
     const removedKey = wordIndexKey(filePath);
+    const removedId = index.fileTable.idFor(removedKey);
+    if (removedId === undefined)
+        return false;
     for (const token of tokenLineCounts.keys()) {
-        const arr = index.postings.get(token);
-        if (!arr)
+        const list = index.postings.get(token);
+        if (!list)
             continue;
-        const next = arr.filter((hit) => wordIndexKey(hit.file) !== removedKey);
-        if (next.length > 0)
+        const next = list.withoutFile(removedId);
+        // `withoutFile` always returns a freshly allocated private store (#2117).
+        if (next.length > 0) {
             index.postings.set(token, next);
+            notePostingStoresAllocated(index, 1);
+        }
         else
             index.postings.delete(token);
     }
@@ -277,6 +528,8 @@ export function removeWordIndexDocument(index, filePath) {
     index.forward.delete(filePath);
     index.fileMtimes.delete(filePath);
     index.fileSizes.delete(filePath);
+    index.dirtyFiles?.add(wordIndexKey(filePath));
+    index.fileTable.release(removedKey);
     index.totalTokens -= docLength;
     index.docCount = Math.max(0, index.docCount - 1);
     return true;
@@ -297,6 +550,7 @@ export function removeWordIndexDocument(index, filePath) {
 export function updateWordIndexDocument(index, doc) {
     if (!index.forward)
         return false;
+    const startedAt = Date.now();
     // Remove the old contribution first (no-op if this is a brand new doc).
     if (index.forward.has(doc.path)) {
         removeWordIndexDocument(index, doc.path);
@@ -322,34 +576,27 @@ export function updateWordIndexDocument(index, doc) {
                 perTokenHits.set(token, [i + 1]);
         }
     }
-    const tokenLineCounts = new Map();
-    for (const [token, lineNumbers] of perTokenHits) {
-        tokenLineCounts.set(token, lineNumbers.length);
-        const hits = lineNumbers.map((line) => ({ file: doc.path, line }));
-        const arr = index.postings.get(token);
-        if (arr)
-            arr.push(...hits);
-        else
-            index.postings.set(token, hits);
-    }
-    index.docLengths.set(doc.path, docLength);
-    index.forward.set(doc.path, tokenLineCounts);
     // Per-edit callers generally already have content but not a stat. -1 is an
     // impossible real mtime, so it deliberately makes the document stale at the
     // next startup refresh (`-1 !== realMtime` always) — unlike 0, which is a
     // legal on-disk mtime (SOURCE_DATE_EPOCH=0, archive extraction) and would
     // collide, leaving such a file never re-tokenized (#958 review F2).
-    index.fileMtimes.set(doc.path, -1);
-    // Size is likewise recorded for the #1105 mtime+size refresh gate. The -1
-    // mtime already forces a re-read next session, so this value only keeps the
-    // map dense (parallel to fileMtimes); store the real byte length so a
-    // deserialize→reserialize round-trip before any refresh carries a truthful
-    // size rather than a placeholder.
-    index.fileSizes.set(doc.path, Buffer.byteLength(doc.content, "utf-8"));
-    index.totalTokens += docLength;
-    index.docCount += 1;
+    commitWordIndexDocumentReplacement(index, doc, perTokenHits, docLength, startedAt);
+    scheduleWordIndexRecompact(index, doc.path);
     return true;
 }
+const asyncWordIndexOperations = new WeakMap();
+function enqueueAsyncWordIndexOperation(index, operation) {
+    const previous = asyncWordIndexOperations.get(index) ?? Promise.resolve();
+    const run = previous.catch(() => undefined).then(operation);
+    const settled = run.then(() => undefined, () => undefined);
+    asyncWordIndexOperations.set(index, settled);
+    return run.finally(() => {
+        if (asyncWordIndexOperations.get(index) === settled) {
+            asyncWordIndexOperations.delete(index);
+        }
+    });
+}
 async function stageWordIndexDocumentRemoval(index, filePath, shouldContinue) {
     if (!index.forward)
         return undefined;
@@ -357,31 +604,43 @@ async function stageWordIndexDocumentRemoval(index, filePath, shouldContinue) {
     if (!tokenLineCounts)
         return undefined;
     const removedKey = wordIndexKey(filePath);
+    const removedId = index.fileTable.idFor(removedKey);
+    if (removedId === undefined)
+        return undefined;
     const postings = new Map();
     const deadline = createDeadline(WORD_INDEX_BUILD_YIELD_BUDGET_MS);
     for (const token of tokenLineCounts.keys()) {
         if (!shouldContinue())
             throw new Error("word index refresh superseded");
-        const arr = index.postings.get(token);
-        if (!arr)
+        const list = index.postings.get(token);
+        if (!list)
             continue;
-        const next = [];
-        for (const hit of arr) {
-            if (wordIndexKey(hit.file) !== removedKey)
-                next.push(hit);
-            if (deadline.expired() && (await yieldIfOverBudget(deadline))) {
-                if (!shouldContinue())
-                    throw new Error("word index refresh superseded");
-            }
+        // One token's postings, filtered by the SAME packed primitive the
+        // synchronous path uses, so the staged result is identical rather than
+        // merely equivalent. The work unit is the token, not the posting element:
+        // a per-element deadline check cost one `performance.now()` per entry —
+        // on the order of a million per edit on a 2.2M-posting corpus — which is
+        // the tax that made this path unusable from the per-edit seam (#2067).
+        // One list's scan is a bounded, non-yieldable unit; the deadline is
+        // checked between lists.
+        const survivors = list.withoutFile(removedId);
+        postings.set(token, survivors.length > 0 ? survivors : undefined);
+        if (deadline.expired() && (await yieldIfOverBudget(deadline))) {
+            if (!shouldContinue())
+                throw new Error("word index refresh superseded");
         }
-        postings.set(token, next.length > 0 ? next : undefined);
     }
     return { postings, docLength: index.docLengths.get(filePath) ?? 0 };
 }
 function commitWordIndexDocumentRemoval(index, filePath, staged) {
-    for (const [token, hits] of staged.postings) {
-        if (hits)
-            index.postings.set(token, hits);
+    for (const [token, list] of staged.postings) {
+        // Staged survivors come from `WordPostingList.withoutFile` — a fresh
+        // private store each, so the O(1) fragmentation gate must count them
+        // (#2117).
+        if (list) {
+            index.postings.set(token, list);
+            notePostingStoresAllocated(index, 1);
+        }
         else
             index.postings.delete(token);
     }
@@ -389,23 +648,49 @@ function commitWordIndexDocumentRemoval(index, filePath, staged) {
     index.forward?.delete(filePath);
     index.fileMtimes.delete(filePath);
     index.fileSizes.delete(filePath);
+    index.dirtyFiles?.add(wordIndexKey(filePath));
+    index.fileTable.release(wordIndexKey(filePath));
     index.totalTokens -= staged.docLength;
     index.docCount = Math.max(0, index.docCount - 1);
 }
 /** Cooperative, atomically-published removal used only by bulk refresh. */
 export async function removeWordIndexDocumentAsync(index, filePath, shouldContinue = () => true) {
-    const staged = await stageWordIndexDocumentRemoval(index, filePath, shouldContinue);
-    if (!staged)
-        return false;
-    if (!shouldContinue())
-        throw new Error("word index refresh superseded");
-    commitWordIndexDocumentRemoval(index, filePath, staged);
-    return true;
+    return enqueueAsyncWordIndexOperation(index, async () => {
+        const staged = await stageWordIndexDocumentRemoval(index, filePath, shouldContinue);
+        if (!staged)
+            return false;
+        if (!shouldContinue())
+            throw new Error("word index refresh superseded");
+        commitWordIndexDocumentRemoval(index, filePath, staged);
+        return true;
+    });
+}
+/**
+ * The per-edit replacement primitive the cascade seam calls (#2067).
+ *
+ * `updateWordIndexDocumentAsync` plus the arena-recompaction gate the
+ * synchronous variant carries, which is what makes this the per-edit entry
+ * point rather than a rename. Bulk refresh must NOT use it: one refreshed
+ * document raises `postingStoreCount` by roughly its distinct-token count, so a
+ * per-document schedule would drive an O(vocabulary) recompaction between
+ * nearly every pair of documents. `refreshWordIndexIncrementally` therefore
+ * drives `updateWordIndexDocumentAsync` directly and recompacts ONCE after its
+ * loop; the per-edit seam has no such "after the loop" and schedules here.
+ */
+export async function updateWordIndexDocumentForEdit(index, doc) {
+    const updated = await updateWordIndexDocumentAsync(index, doc);
+    if (updated)
+        scheduleWordIndexRecompact(index, doc.path);
+    return updated;
 }
 /** Cooperative replacement whose old/new state is committed without an await. */
 export async function updateWordIndexDocumentAsync(index, doc, shouldContinue = () => true) {
+    return enqueueAsyncWordIndexOperation(index, () => updateWordIndexDocumentAsyncUnsafe(index, doc, shouldContinue));
+}
+async function updateWordIndexDocumentAsyncUnsafe(index, doc, shouldContinue) {
     if (!index.forward)
         return false;
+    const startedAt = Date.now();
     const removal = index.forward.has(doc.path)
         ? await stageWordIndexDocumentRemoval(index, doc.path, shouldContinue)
         : undefined;
@@ -437,22 +722,7 @@ export async function updateWordIndexDocumentAsync(index, doc, shouldContinue =
         throw new Error("word index refresh superseded");
     if (removal)
         commitWordIndexDocumentRemoval(index, doc.path, removal);
-    const tokenLineCounts = new Map();
-    for (const [token, lineNumbers] of perTokenHits) {
-        tokenLineCounts.set(token, lineNumbers.length);
-        const hits = lineNumbers.map((line) => ({ file: doc.path, line }));
-        const arr = index.postings.get(token);
-        if (arr)
-            arr.push(...hits);
-        else
-            index.postings.set(token, hits);
-    }
-    index.docLengths.set(doc.path, docLength);
-    index.forward.set(doc.path, tokenLineCounts);
-    index.fileMtimes.set(doc.path, -1);
-    index.fileSizes.set(doc.path, Buffer.byteLength(doc.content, "utf-8"));
-    index.totalTokens += docLength;
-    index.docCount += 1;
+    commitWordIndexDocumentReplacement(index, doc, perTokenHits, docLength, startedAt);
     return true;
 }
 /** Bounds shared by every word-index build path — keep the walk off the
@@ -567,6 +837,12 @@ const WORD_INDEX_FILE_READ_TOKEN_COST = 300;
  * against a 1,436 ms rebuild + re-read). The synthetic high-df corpus in the
  * #1197 probe gives 0.09, so this is the conservative (incremental-favouring)
  * end of the observed range.
+ *
+ * #2069 made the scan an Int32 compare over packed lanes instead of a string
+ * compare over boxed objects, so the real per-element cost is now BELOW this
+ * constant. The constant is left as measured: over-stating incremental cost
+ * only ever routes work to a full rebuild, which is the bounded side of the
+ * decision. Re-measuring it is #2067/#2068 territory, not this change.
  */
 const WORD_INDEX_POSTING_SCAN_TOKEN_COST = 0.2;
 /**
@@ -703,9 +979,9 @@ export async function refreshWordIndexIncrementally(index, root, shouldContinue
     // this decision exists to avoid.
     let postingEntries = 0;
     let weightedPostingEntries = 0;
-    for (const hits of index.postings.values()) {
-        postingEntries += hits.length;
-        weightedPostingEntries += hits.length * hits.length;
+    for (const list of index.postings.values()) {
+        postingEntries += list.length;
+        weightedPostingEntries += list.length * list.length;
     }
     // The mean posting-array length weighted by OCCURRENCES, not the plain mean:
     // a document's tokens are drawn from the frequency distribution, so the arrays
@@ -802,6 +1078,9 @@ export async function refreshWordIndexIncrementally(index, root, shouldContinue
         }
     }
     timings.refreshReadsMs = Date.now() - refreshReadsStartMs;
+    // Route through the same guarded queue the per-edit seam uses, so a refresh
+    // and a concurrent cascade edit cannot both drive a recompaction (#2117).
+    await enqueueWordIndexRecompact(index, root);
     index.truncated = walked.length === maxFiles;
     return {
         mode: "incremental",
@@ -1026,17 +1305,29 @@ export function searchWordIndex(index, query, options = {}) {
         const posting = index.postings.get(token);
         if (!posting)
             continue;
-        const linesByFile = new Map();
-        for (const hit of posting) {
-            const arr = linesByFile.get(hit.file);
+        // Grouped by the packed file id, then resolved to the display path once
+        // per (token, file) instead of once per posting (#2069). Insertion order,
+        // document frequency, and the per-file line lists are identical to the
+        // boxed representation, so scores and ordering are unchanged.
+        const linesByFileId = new Map();
+        for (let i = 0; i < posting.length; i += 1) {
+            const fileId = posting.fileIdAt(i);
+            const arr = linesByFileId.get(fileId);
             if (arr)
-                arr.push(hit.line);
+                arr.push(posting.lineAt(i));
             else
-                linesByFile.set(hit.file, [hit.line]);
+                linesByFileId.set(fileId, [posting.lineAt(i)]);
         }
-        const docFrequency = linesByFile.size;
+        const docFrequency = linesByFileId.size;
         const idf = Math.log(1 + (docCount - docFrequency + 0.5) / (docFrequency + 0.5));
-        for (const [file, lines] of linesByFile) {
+        for (const [fileId, lines] of linesByFileId) {
+            const file = index.fileTable.pathFor(fileId);
+            if (file === undefined) {
+                // Dropping this silently would shorten the result list with nothing to
+                // tell it apart from a smaller match set. Record, then drop.
+                recordOrphanWordIndexFileId(fileId, token, "search");
+                continue;
+            }
             if (combinedFilter && !combinedFilter(file))
                 continue;
             const termFrequency = lines.length;
@@ -1100,18 +1391,32 @@ export function centralityFromReverseDeps(index, reverseDeps, normalizeKey = (fi
 }
 /** Persisted word-index serialization format version. Bump on breaking format changes. */
 export const WORD_INDEX_FORMAT_VERSION = 2;
-export function serializeWordIndex(index) {
+const serializedWordIndexCaches = new WeakMap();
+let _lastSerializeWork;
+/** Test-only: work stats for the most recent {@link serializeWordIndex} call. */
+export function getLastWordIndexSerializeWork() {
+    return _lastSerializeWork;
+}
+function serializeWordIndexFull(index) {
     const files = [...index.docLengths.keys()];
-    const fileIndex = new Map();
-    files.forEach((file, i) => fileIndex.set(file, i));
+    // `files` carries whatever spelling `docLengths` last stored, which can
+    // differ from the spelling the file table interned. Both fold through
+    // `wordIndexKey`, so the slot lookup runs in the folded key space and a
+    // separator- or case-divergent spelling still finds its postings (#1025).
+    const slotByFileId = new Map();
+    files.forEach((file, i) => {
+        const fileId = index.fileTable.idFor(wordIndexKey(file));
+        if (fileId !== undefined)
+            slotByFileId.set(fileId, i);
+    });
     const postings = [];
-    for (const [token, hits] of index.postings) {
+    for (const [token, list] of index.postings) {
         const flat = [];
-        for (const hit of hits) {
-            const idx = fileIndex.get(hit.file);
-            if (idx === undefined)
+        for (let i = 0; i < list.length; i += 1) {
+            const slot = slotByFileId.get(list.fileIdAt(i));
+            if (slot === undefined)
                 continue;
-            flat.push(idx, hit.line);
+            flat.push(slot, list.lineAt(i));
         }
         if (flat.length > 0)
             postings.push([token, flat]);
@@ -1119,10 +1424,10 @@ export function serializeWordIndex(index) {
     const forward = index.forward
         ? files.map((file, i) => [
             i,
-            [...(index.forward.get(file) ?? new Map()).entries()],
+            [...(index.forward.get(file)?.entries() ?? [])],
         ])
         : undefined;
-    return {
+    const serialized = {
         version: WORD_INDEX_FORMAT_VERSION,
         files,
         postings,
@@ -1134,6 +1439,145 @@ export function serializeWordIndex(index) {
         fileSizes: files.map((file) => index.fileSizes.get(file) ?? 0),
         forward,
     };
+    const tokensByFile = new Map();
+    if (index.forward) {
+        for (const file of files) {
+            tokensByFile.set(wordIndexKey(file), new Set(index.forward.get(file)?.keys() ?? []));
+        }
+    }
+    _lastSerializeWork = {
+        affectedTokenCount: postings.length,
+        tookFullPath: true,
+    };
+    return { serialized, slotByFileId, tokensByFile };
+}
+/**
+ * Refresh the cached wire view without walking untouched posting lanes.
+ * Existing files keep their slots, so a document replacement only rebuilds
+ * the token lists named by that document's old or new forward entry. A file
+ * removal or reordering changes slot identity and deliberately takes the
+ * bounded full path.
+ */
+function serializeWordIndexIncrementally(index, cache) {
+    const files = [...index.docLengths.keys()];
+    const priorFiles = cache.serialized.files;
+    const currentKeys = new Set(files.map(wordIndexKey));
+    const priorKeys = new Set(priorFiles.map(wordIndexKey));
+    if ([...priorKeys].some((key) => !currentKeys.has(key))) {
+        return serializeWordIndexFull(index);
+    }
+    // Re-flattening most lanes costs more than rebuilding the compact wire view.
+    // The review fixture crosses over between 10% and 100% dirty, so use the
+    // midpoint as a conservative bound until a workload-specific model exists.
+    const dirty = index.dirtyFiles;
+    if (dirty && dirty.size * 2 > files.length) {
+        return serializeWordIndexFull(index);
+    }
+    // A replacement deletes and re-adds a PathKeyedMap entry, which changes Map
+    // insertion order. Keep the cached wire slots stable instead of rewriting
+    // every posting just because one document changed order.
+    const filesInWireOrder = [
+        ...priorFiles,
+        ...files.filter((file) => !priorKeys.has(wordIndexKey(file))),
+    ];
+    if (!dirty || dirty.size === 0) {
+        _lastSerializeWork = { affectedTokenCount: 0, tookFullPath: false };
+        return cache;
+    }
+    const slotByFileId = new Map(cache.slotByFileId);
+    for (let i = priorFiles.length; i < filesInWireOrder.length; i += 1) {
+        const fileId = index.fileTable.idFor(wordIndexKey(filesInWireOrder[i]));
+        if (fileId !== undefined)
+            slotByFileId.set(fileId, i);
+    }
+    const postings = cache.serialized.postings.slice();
+    const postingAt = new Map();
+    postings.forEach(([token], i) => postingAt.set(token, i));
+    const removedTokens = new Set();
+    const tokensByFile = new Map(cache.tokensByFile);
+    const fileByKey = new Map(filesInWireOrder.map((file) => [wordIndexKey(file), file]));
+    const affectedTokens = new Set();
+    for (const fileKey of dirty) {
+        const file = fileByKey.get(fileKey);
+        const oldTokens = tokensByFile.get(fileKey) ?? new Set();
+        const currentEntry = file === undefined ? undefined : index.forward?.get(file);
+        const currentTokens = new Set(currentEntry?.keys() ?? []);
+        for (const token of oldTokens)
+            affectedTokens.add(token);
+        for (const token of currentTokens)
+            affectedTokens.add(token);
+        if (file === undefined)
+            tokensByFile.delete(fileKey);
+        else
+            tokensByFile.set(fileKey, currentTokens);
+    }
+    for (const token of affectedTokens) {
+        const list = index.postings.get(token);
+        const flat = [];
+        if (list) {
+            for (let i = 0; i < list.length; i += 1) {
+                const slot = slotByFileId.get(list.fileIdAt(i));
+                if (slot !== undefined)
+                    flat.push(slot, list.lineAt(i));
+            }
+        }
+        const at = postingAt.get(token);
+        if (flat.length === 0) {
+            if (at !== undefined)
+                removedTokens.add(token);
+        }
+        else if (at === undefined) {
+            postings.push([token, flat]);
+            postingAt.set(token, postings.length - 1);
+        }
+        else {
+            postings[at] = [postings[at][0], flat];
+        }
+    }
+    const compactedPostings = removedTokens.size
+        ? postings.filter(([token]) => !removedTokens.has(token))
+        : postings;
+    const forward = index.forward
+        ? (cache.serialized.forward
+            ?.slice()
+            .map((entry) => [entry[0], entry[1]]) ?? [])
+        : undefined;
+    for (const fileKey of dirty) {
+        const file = fileByKey.get(fileKey);
+        const fileId = file === undefined
+            ? undefined
+            : index.fileTable.idFor(wordIndexKey(file));
+        const slot = fileId === undefined ? undefined : slotByFileId.get(fileId);
+        if (slot === undefined || !forward || file === undefined)
+            continue;
+        forward[slot] = [slot, [...(index.forward?.get(file)?.entries() ?? [])]];
+    }
+    const serialized = {
+        ...cache.serialized,
+        files: filesInWireOrder,
+        postings: compactedPostings,
+        docLengths: filesInWireOrder.map((file) => index.docLengths.get(file) ?? 0),
+        fileMtimes: filesInWireOrder.map((file) => index.fileMtimes.get(file) ?? 0),
+        fileSizes: filesInWireOrder.map((file) => index.fileSizes.get(file) ?? 0),
+        forward,
+        totalTokens: index.totalTokens,
+        indexedFileCount: index.docCount,
+        truncated: index.truncated,
+    };
+    _lastSerializeWork = {
+        affectedTokenCount: affectedTokens.size,
+        tookFullPath: false,
+    };
+    return { serialized, slotByFileId, tokensByFile };
+}
+export function serializeWordIndex(index) {
+    const prior = serializedWordIndexCaches.get(index);
+    const cache = prior
+        ? serializeWordIndexIncrementally(index, prior)
+        : serializeWordIndexFull(index);
+    serializedWordIndexCaches.set(index, cache);
+    index.dirtyFiles?.clear();
+    return cache.serialized;
 }
 export function deserializeWordIndex(data) {
     if (!data ||
@@ -1146,6 +1590,12 @@ export function deserializeWordIndex(data) {
         return null;
     }
     const docLengths = new PathKeyedMap(wordIndexKey);
+    // Slot i of the wire `files` array becomes file id `fileIdBySlot[i]`. Two
+    // slots whose spellings fold to one key share one id, exactly as a build
+    // would intern them, so a divergent snapshot cannot deserialize into two
+    // posting identities for one document.
+    const fileTable = new WordIndexFileTable();
+    const fileIdBySlot = data.files.map((file) => fileTable.intern(wordIndexKey(file), file));
     const fileMtimes = new PathKeyedMap(wordIndexKey);
     const fileSizes = new PathKeyedMap(wordIndexKey);
     data.files.forEach((file, i) => docLengths.set(file, data.docLengths[i] ?? 0));
@@ -1162,17 +1612,23 @@ export function deserializeWordIndex(data) {
     for (const [token, flat] of data.postings) {
         if (typeof token !== "string" || !Array.isArray(flat))
             continue;
-        const hits = [];
+        const lanes = [];
         for (let i = 0; i + 1 < flat.length; i += 2) {
-            const file = data.files[flat[i]];
+            const fileId = fileIdBySlot[flat[i]];
             const line = flat[i + 1];
-            if (typeof file === "string" && typeof line === "number") {
-                hits.push({ file, line });
+            if (typeof fileId === "number" && typeof line === "number") {
+                lanes.push(fileId, line);
             }
         }
-        if (hits.length > 0)
-            postings.set(token, hits);
+        if (lanes.length > 0) {
+            postings.set(token, WordPostingList.fromLanes(token, lanes));
+        }
     }
+    // Every list above was sized exactly from the wire payload, so there is no
+    // growth slack to release — but the per-list `ArrayBuffer` headers are the
+    // same third-of-the-store cost a fresh build pays, and a deserialized index
+    // is the LONG-lived one (a warm session reuses it across turns).
+    compactPostingsIntoArena(postings);
     let forward;
     if (Array.isArray(data.forward)) {
         forward = new PathKeyedMap(wordIndexKey);
@@ -1189,14 +1645,17 @@ export function deserializeWordIndex(data) {
                     continue;
                 const [token, count] = pair;
                 if (typeof token === "string" && typeof count === "number") {
-                    perToken.set(token, count);
+                    // Point at the canonical instance the postings map holds, not
+                    // this document’s own JSON.parse allocation (#2069).
+                    perToken.set(postings.get(token)?.token ?? token, count);
                 }
             }
-            forward.set(file, perToken);
+            forward.set(file, WordForwardEntry.fromTally(perToken));
         }
     }
     return {
         postings,
+        fileTable,
         docLengths,
         totalTokens: typeof data.totalTokens === "number" ? data.totalTokens : 0,
         docCount: data.files.length,
@@ -1204,6 +1663,10 @@ export function deserializeWordIndex(data) {
         forward,
         fileMtimes,
         fileSizes,
+        // Postings were just packed into one arena above, so the running gate
+        // starts from the exact post-compaction store count (#2117).
+        postingStoreCount: countPostingBackingStores(postings),
+        recompactFlight: createSingleFlight(),
     };
 }
 const buildStatuses = new Map();
@@ -1278,6 +1741,8 @@ export function triggerBackgroundWordIndexBuild(cwd, dbg, options = {}) {
                 durationMs: Date.now() - startMs,
                 indexedFileCount: index.docCount,
                 tokens: index.postings.size,
+                postingEntries: countWordIndexPostingEntries(index),
+                residentBytes: estimateWordIndexResidentBytes(index),
                 truncated: index.truncated,
                 skipped: docs.skipped,
             });
@@ -1332,6 +1797,7 @@ function getWordIndexPersistScheduler() {
     return wordIndexPersistScheduler;
 }
 async function writeWordIndexSnapshot(cwd, index, dbg) {
+    const persistStartedAt = Date.now();
     try {
         const { loadProjectSnapshot, saveProjectSnapshot, PROJECT_SNAPSHOT_VERSION, } = await import("./project-snapshot.js");
         const existing = loadProjectSnapshot(cwd);
@@ -1346,8 +1812,11 @@ async function writeWordIndexSnapshot(cwd, index, dbg) {
             cachedExports: [],
         };
         snapshot.generatedAt = new Date().toISOString();
+        const serializeStartedAt = performance.now();
         snapshot.wordIndex = serializeWordIndex(index);
+        const serializeMs = performance.now() - serializeStartedAt;
         saveProjectSnapshot(cwd, snapshot);
+        const writeMs = performance.now() - serializeStartedAt - serializeMs;
         dbg?.(`word-index persist: ${index.docCount} files, ${index.postings.size} tokens`);
         // #958 review F1: durably record persist SUCCESS too, not just failures —
         // in the MCP host (`dbg` is a no-op) this is the only signal that the index
@@ -1357,9 +1826,18 @@ async function writeWordIndexSnapshot(cwd, index, dbg) {
             phase: "persist_succeeded",
             cwd: path.resolve(cwd),
             trigger: "per_edit",
+            durationMs: Date.now() - persistStartedAt,
+            serializeMs,
+            writeMs,
             indexedFileCount: index.docCount,
             tokens: index.postings.size,
+            postingEntries: countWordIndexPostingEntries(index),
+            residentBytes: estimateWordIndexResidentBytes(index),
+            replacementCount: index.replacementStats?.count ?? 0,
+            totalReplacementMs: index.replacementStats?.totalMs ?? 0,
+            maxReplacementMs: index.replacementStats?.maxMs ?? 0,
         });
+        index.replacementStats = { count: 0, totalMs: 0, maxMs: 0 };
     }
     catch (err) {
         dbg?.(`word-index persist: failed: ${err}`);