cgraphx 2.0.2 → 2.0.3

package/dist/core/code/engine/storage/parsedfile-store.js

@@ -0,0 +1,620 @@
+"use strict";
+/**
+ * Disk-backed `ParsedFile` store (#1983 scope-resolution OOM).
+ *
+ * ## Why this exists
+ *
+ * The scope-resolution phase needs a `ParsedFile` (scopes / defs / reference
+ * sites) for every file. Historically it re-extracted each file from source on
+ * the **main thread** via `extractParsedFile` → `parseSourceSafe`. On a huge
+ * repo (Linux kernel, ~64k C files) that re-parse accumulates an unbounded
+ * **native** memory leak in `tree-sitter` 0.21.1 (`CallbackInput` retains the
+ * input string with no destructor; node-tree-sitter PR #201) — the leaked
+ * `TSTree` memory is invisible to V8, never reclaimed by GC, and not freed by
+ * worker_thread teardown. The parse phase escapes it only because each parse is
+ * relatively cheap there; a second full re-parse of every file on the immortal
+ * main thread pushes RSS past the heap cap and the OOM-killer fires.
+ *
+ * The fix: the parse workers already build a tree-sitter `Tree` per file, so
+ * they emit the `ParsedFile` directly (reusing that tree — no second parse).
+ * Holding all of them in main-thread heap is what the original #1983 work
+ * removed (it cost ~1× the semantic model in RAM during parse), so instead we
+ * flush them to this disk store per chunk and stream them back per language in
+ * scope-resolution. Net effect: the file is parsed exactly once (in a worker),
+ * scope-resolution does ZERO parsing, and peak heap stays bounded.
+ *
+ * ## Shape
+ *
+ * `<storagePath>/parsedfile-store/<shardId>.json` — one shard per parse chunk,
+ * a JSON array of `ParsedFile` serialized with the same `mapReplacer` the parse
+ * cache uses (Scope.bindings / Scope.typeBindings are `Map`s). The store is
+ * cleared at the start of each parse and after scope-resolution consumes it, so
+ * it never lingers and never goes stale across runs.
+ *
+ * ## Durable sibling store (`parsedfile-cache/`, warm-cache coverage)
+ *
+ * The run-scoped store above is only populated when the parse workers actually
+ * run. On a warm re-analyze where every chunk is a parse-cache HIT, no worker
+ * runs, the run-scoped store was cleared at parse start, and the cached
+ * `ParseWorkerResult` carries no `ParsedFile`s (the worker emptied them after
+ * its store write) — so scope-resolution would find an empty store and fall
+ * back to main-thread `extractParsedFile`, re-opening the #1983 OOM. To close
+ * that gap we ALSO write the worker's ParsedFiles to a second, CONTENT-ADDRESSED
+ * store keyed by the parse chunk hash (`getDurableParsedFileDir`), which mirrors
+ * the parse cache's lifecycle (persists across runs, pruned by `usedKeys`,
+ * version-tied via `PARSE_CACHE_VERSION`). On a warm hit the chunk's durable
+ * shards are byte-COPIED into the run-scoped store (no re-parse, no
+ * re-serialize → byte-identical), so scope-resolution streams them exactly as
+ * on a cold run. Content-addressing makes stale reuse impossible: a changed
+ * file changes its chunk hash, which misses BOTH stores and re-dispatches.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pruneAndSaveDurableParsedFileStore = exports.loadDurableParsedFileIndex = exports.restoreDurableParsedFileShard = exports.persistDurableParsedFileShardSync = exports.prepareDurableParsedFileChunk = exports.getDurableParsedFileDir = exports.loadParsedFilesForPaths = exports.persistParsedFileShardSync = exports.persistParsedFileChunk = exports.clearParsedFileStore = exports.getParsedFileStoreDir = exports.forceGc = exports.makeInterningReviver = void 0;
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const node_v8_1 = __importDefault(require("node:v8"));
+const node_vm_1 = __importDefault(require("node:vm"));
+const receiver_chain_codec_js_1 = require("../core/ingestion/utils/receiver-chain-codec.js");
+const logger_js_1 = require("../core/logger.js");
+const parse_cache_js_1 = require("./parse-cache.js");
+const STORE_DIRNAME = 'parsedfile-store';
+const DURABLE_DIRNAME = 'parsedfile-cache';
+const DURABLE_INDEX_FILENAME = 'index.json';
+const MAX_CALLABLE_FLOW_SITES_PER_FILE = 100_000;
+const MAX_CALLABLE_FLOW_INDIRECTION = 16;
+const MAX_CALLABLE_FLOW_NAME_LENGTH = 4_096;
+const MAX_CALLABLE_FLOW_PARAMETERS = 1_024;
+/**
+ * Build a JSON.parse reviver that (a) interns every string against a shared
+ * pool and (b) applies the parse-cache `mapReviver` (Map/Set reconstruction).
+ *
+ * `JSON.parse` allocates a DISTINCT string object for every textual token, so a
+ * `ParsedFile` graph round-tripped through disk holds millions of duplicate
+ * strings — every def repeats its `filePath`, and common type/qualified names
+ * (`int`, `void`, `struct …`) recur across the whole repo. On the Linux kernel
+ * that roughly DOUBLES the deserialized heap (~15 GB vs ~7.6 GB interned).
+ * Interning IN the reviver collapses duplicates as the tree is revived (one
+ * pass, no second walk). The pool is per-load; the interned strings stay shared
+ * through the retained `ParsedFile` references after the pool is dropped.
+ */
+const makeInterningReviver = (pool, defPool) => {
+    return (key, value) => {
+        if (typeof value === 'string') {
+            const hit = pool.get(value);
+            if (hit !== undefined)
+                return hit;
+            pool.set(value, value);
+            return value;
+        }
+        const revived = (0, parse_cache_js_1.mapReviver)(key, value);
+        // Collapse the THREE serialized copies of each `SymbolDefinition` back to one
+        // shared object, keyed on the def-exclusive `nodeId`. A def is serialized in
+        // `localDefs`, in its owning `scope.ownedDefs`, and inside `scope.bindings[].def`;
+        // in the live extractor those are ONE object by reference, but `JSON.parse`
+        // rebuilds three distinct objects (string interning alone leaves the object
+        // duplication intact — ~3× the def-object heap on the disk-backed path).
+        // Re-sharing is byte-identical to resolution: every consumer reads defs BY
+        // VALUE (`nodeId`/`type`), never by object identity, and the authoritative
+        // resolver index is built from `localDefs` only. `nodeId` is def-exclusive in
+        // the scope-resolution schema; the `filePath` check guards future shapes.
+        if (revived !== null &&
+            typeof revived === 'object' &&
+            typeof revived.nodeId === 'string' &&
+            typeof revived.filePath === 'string') {
+            const def = revived;
+            const seen = defPool.get(def.nodeId);
+            if (seen !== undefined)
+                return seen;
+            defPool.set(def.nodeId, def);
+            return def;
+        }
+        return revived;
+    };
+};
+exports.makeInterningReviver = makeInterningReviver;
+/**
+ * Best-effort forced garbage collection. `JSON.parse` of each shard builds a
+ * transient bloated (pre-intern) tree; across hundreds of shards that churn
+ * outpaces V8's incremental GC and piles up against the heap limit (measured
+ * ~5 GB of avoidable transient on the kernel). A periodic full GC during the
+ * load keeps the peak at the retained set rather than retained + churn. Uses
+ * the global `gc` when exposed, else the v8/vm trick — and degrades to a no-op
+ * if neither is available, so it never throws.
+ */
+let cachedGc;
+/**
+ * Best-effort synchronous GC. Uses `globalThis.gc` when `--expose-gc` is set,
+ * else lazily wires it via `v8.setFlagsFromString('--expose-gc')` + a fresh
+ * `vm` context. Exported so scope-resolution can reclaim a finished language's
+ * ParsedFiles at the per-language eviction boundary (#1741 / kernel memory work).
+ */
+const forceGc = () => {
+    const g = globalThis.gc;
+    if (typeof g === 'function') {
+        g();
+        return;
+    }
+    if (cachedGc === undefined) {
+        cachedGc = null;
+        try {
+            node_v8_1.default.setFlagsFromString('--expose-gc');
+            cachedGc = node_vm_1.default.runInNewContext('gc');
+            node_v8_1.default.setFlagsFromString('--no-expose-gc');
+        }
+        catch {
+            cachedGc = null;
+        }
+    }
+    cachedGc?.();
+};
+exports.forceGc = forceGc;
+const getParsedFileStoreDir = (storagePath) => node_path_1.default.join(storagePath, STORE_DIRNAME);
+exports.getParsedFileStoreDir = getParsedFileStoreDir;
+/** Remove any prior run's shards so a fresh parse starts clean. Idempotent. */
+const clearParsedFileStore = async (storagePath) => {
+    await node_fs_1.promises.rm((0, exports.getParsedFileStoreDir)(storagePath), { recursive: true, force: true });
+};
+exports.clearParsedFileStore = clearParsedFileStore;
+/**
+ * Single source of truth for a shard's bytes. Returns `null` for an empty
+ * chunk (caller writes nothing). Both the async (`persistParsedFileChunk`) and
+ * sync (`persistParsedFileShardSync`) writers go through this so the two paths
+ * are guaranteed byte-identical — the shards must round-trip through the same
+ * `mapReviver`, and matching bytes by having both authors type the same
+ * `mapReplacer` call would be a coincidence, not a guarantee.
+ */
+const serializeParsedFileShard = (parsedFiles) => {
+    if (parsedFiles.length === 0)
+        return null;
+    return JSON.stringify(parsedFiles, parse_cache_js_1.mapReplacer);
+};
+const shardPath = (storagePath, shardId) => node_path_1.default.join((0, exports.getParsedFileStoreDir)(storagePath), `${shardId}.json`);
+/**
+ * Write one parse chunk's `ParsedFile[]` to the store as a single shard (async).
+ * No-op for an empty chunk. `shardId` must be unique within a run. Used by the
+ * main-thread no-store-disabled fallback and any non-worker writer; the worker
+ * store path uses {@link persistParsedFileShardSync}.
+ */
+const persistParsedFileChunk = async (storagePath, shardId, parsedFiles) => {
+    const payload = serializeParsedFileShard(parsedFiles);
+    if (payload === null)
+        return;
+    await node_fs_1.promises.mkdir((0, exports.getParsedFileStoreDir)(storagePath), { recursive: true });
+    await node_fs_1.promises.writeFile(shardPath(storagePath, shardId), payload, 'utf-8');
+};
+exports.persistParsedFileChunk = persistParsedFileChunk;
+// Per-process set of store dirs we've already `mkdir`ed, so the sync worker
+// writer (called once per job, many times into the same dir) doesn't issue a
+// `mkdirSync` syscall on every shard. Mirrors parse-cache.ts's `createdCacheDirs`.
+const createdStoreDirs = new Set();
+/**
+ * Synchronous shard writer for use INSIDE a parse worker (#1983 parallel
+ * serialization). The worker is a dedicated thread, so a blocking write there
+ * protects the main thread, and a sync write avoids threading `async`/`await`
+ * through the synchronous per-file extract loop. Produces byte-identical shards
+ * to {@link persistParsedFileChunk} via the shared {@link serializeParsedFileShard}.
+ * No-op for an empty chunk. `shardId` must be globally unique for the run (the
+ * worker uses `w<threadId>-<seq>`); a duplicate would silently overwrite.
+ */
+const persistParsedFileShardSync = (storagePath, shardId, parsedFiles) => {
+    const payload = serializeParsedFileShard(parsedFiles);
+    if (payload === null)
+        return;
+    const dir = (0, exports.getParsedFileStoreDir)(storagePath);
+    if (!createdStoreDirs.has(dir)) {
+        (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+        createdStoreDirs.add(dir);
+    }
+    (0, node_fs_1.writeFileSync)(shardPath(storagePath, shardId), payload, 'utf-8');
+};
+exports.persistParsedFileShardSync = persistParsedFileShardSync;
+/**
+ * Stream the store and return the `ParsedFile`s whose `filePath` is in
+ * `wantPaths`, keyed by path. Loads one shard at a time and retains only the
+ * matching entries, so peak heap is bounded by (matched set) + (one shard)
+ * rather than the whole store. Returns an empty map when the store is absent
+ * (e.g. tests, or a run with no worker pool) — callers fall back to a fresh
+ * extract for the missing files.
+ */
+const loadParsedFilesForPaths = async (storagePath, wantPaths) => {
+    const out = new Map();
+    if (wantPaths.size === 0)
+        return out;
+    const dir = (0, exports.getParsedFileStoreDir)(storagePath);
+    let shards;
+    try {
+        shards = (await node_fs_1.promises.readdir(dir)).filter((f) => f.endsWith('.json'));
+    }
+    catch {
+        return out; // store absent
+    }
+    // Shared interning pool for this load — deduplicates strings ACROSS shards
+    // (one `int` / one repeated filePath for the whole language), which is where
+    // most of the saving comes from. Dropped when this function returns.
+    const pool = new Map();
+    let droppedSites = 0;
+    let filesWithDroppedSites = 0;
+    let droppedChains = 0;
+    let rejectedFiles = 0;
+    for (let i = 0; i < shards.length; i++) {
+        // Per-shard def pool: a SymbolDefinition's three serialized copies live within
+        // a single shard (one ParsedFile), so the dedup is shard-local. A cross-shard
+        // pool would retain defs of files NOT in `wantPaths` (loaded-but-discarded
+        // shards), reintroducing the leak; per-shard drops them with the shard.
+        const defPool = new Map();
+        const reviver = (0, exports.makeInterningReviver)(pool, defPool);
+        let parsed;
+        try {
+            const raw = await node_fs_1.promises.readFile(node_path_1.default.join(dir, shards[i]), 'utf-8');
+            parsed = JSON.parse(raw, reviver);
+        }
+        catch {
+            continue; // skip a corrupt shard; missing files fall back to fresh extract
+        }
+        if (!Array.isArray(parsed))
+            continue;
+        for (const pf of parsed) {
+            if (!pf || typeof pf.filePath !== 'string' || !wantPaths.has(pf.filePath))
+                continue;
+            const flow = sanitizeCallableFlowSites(pf.callableFlowSites);
+            if (flow === undefined) {
+                // non-array garbage → distrust the file, re-extract
+                rejectedFiles++;
+                continue;
+            }
+            const chains = sanitizeReceiverChains(pf.referenceSites);
+            if (chains === undefined) {
+                rejectedFiles++;
+                continue;
+            }
+            if (flow.dropped === 0 && chains.dropped === 0) {
+                out.set(pf.filePath, pf);
+            }
+            else {
+                droppedSites += flow.dropped;
+                droppedChains += chains.dropped;
+                filesWithDroppedSites++;
+                out.set(pf.filePath, {
+                    ...pf,
+                    ...(flow.dropped === 0 ? {} : { callableFlowSites: flow.sites }),
+                    ...(chains.dropped === 0 ? {} : { referenceSites: chains.sites }),
+                });
+            }
+        }
+        // Every few shards, reclaim the transient pre-intern parse churn before it
+        // piles up against the heap limit (~5 GB avoidable on the kernel), and
+        // yield so the GC + any pending I/O can run.
+        if ((i & 7) === 7) {
+            (0, exports.forceGc)();
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    }
+    if (droppedSites > 0 || droppedChains > 0) {
+        // Facts for the dropped sites are omitted this run (the file itself is
+        // retained, so no re-extract happens) — surface it so a recurring drop
+        // on every warm load is observable rather than silent (#2522 review).
+        logger_js_1.logger.warn({ droppedSites, droppedChains, files: filesWithDroppedSites }, 'parsedfile-store: dropped malformed/over-bound sites at load; files retained without those facts');
+    }
+    if (rejectedFiles > 0) {
+        // The other half of the same defect. A rejected file silently falls back to
+        // a fresh extract EVERY load, so a writer that keeps minting what this
+        // reader keeps refusing is a permanent warm-cache miss that costs real time
+        // and says nothing about why.
+        logger_js_1.logger.warn({ rejectedFiles }, 'parsedfile-store: rejected shard entries at load (untrusted shape); those files re-extract every run');
+    }
+    return out;
+};
+exports.loadParsedFilesForPaths = loadParsedFilesForPaths;
+/**
+ * Treat the durable ParsedFile store as an untrusted serialization boundary.
+ * Sanitation is per-SITE, not per-file: one malformed or over-bound fact drops
+ * only itself (counted, logged by the caller), so a legitimately pathological
+ * source file cannot push its whole ParsedFile into a permanent, silent
+ * warm-cache-miss reparse loop (#2522 review). Only a non-array field —
+ * i.e. garbage that says the serialization itself is untrustworthy — rejects
+ * the file, and `undefined` (never emitted / no facts) passes through.
+ * Returns `undefined` for the reject-file case.
+ */
+function sanitizeCallableFlowSites(value) {
+    if (value === undefined)
+        return { sites: undefined, dropped: 0 };
+    if (!Array.isArray(value))
+        return undefined;
+    const bounded = value.length > MAX_CALLABLE_FLOW_SITES_PER_FILE
+        ? value.slice(0, MAX_CALLABLE_FLOW_SITES_PER_FILE)
+        : value;
+    const sites = bounded.filter(isValidCallableFlowSite);
+    return { sites, dropped: value.length - sites.length };
+}
+/**
+ * Untrusted-boundary handling for the compact `receiverChain` on a reference site.
+ *
+ * NARROWER than its sibling `sanitizeCallableFlowSites`, deliberately and worth
+ * stating plainly: that function validates every element with a full type
+ * predicate, whereas this one inspects only the `receiverChain` sub-field and, in
+ * the common case where nothing was stripped, returns the original array typed as
+ * `ReferenceSite[]` without structurally validating `name` / `kind` / `atRange`.
+ * That is still a strict improvement — `referenceSites` had NO sanitation at all
+ * before this — but it is not parity, and a future change that needs per-site
+ * structural validation must add it rather than assume it is already here.
+ *
+ * Sanitation is per-FIELD, not per-site: a chain that does not decode is
+ * stripped and the site is kept, because the site is still perfectly usable
+ * through the text cascade — dropping the whole reference would turn a
+ * degraded receiver into a missing edge. Only a non-array `referenceSites`
+ * rejects the file, and `undefined` (a store written before this field
+ * existed) passes through untouched, which is what makes an old shard load
+ * without error.
+ *
+ * Returns the ORIGINAL array when nothing was stripped, so the overwhelmingly
+ * common case allocates nothing on a hot warm-load path.
+ */
+function sanitizeReceiverChains(value) {
+    if (value === undefined)
+        return { sites: undefined, dropped: 0 };
+    if (!Array.isArray(value))
+        return undefined;
+    let dropped = 0;
+    for (const site of value) {
+        if (isRecord(site) &&
+            site.receiverChain !== undefined &&
+            !(0, receiver_chain_codec_js_1.isValidReceiverChain)(site.receiverChain))
+            dropped++;
+    }
+    if (dropped === 0)
+        return { sites: value, dropped: 0 };
+    const sites = value.map((site) => {
+        if (!isRecord(site) || site.receiverChain === undefined)
+            return site;
+        if ((0, receiver_chain_codec_js_1.isValidReceiverChain)(site.receiverChain))
+            return site;
+        const { receiverChain: _dropped, ...rest } = site;
+        return rest;
+    });
+    return { sites, dropped };
+}
+function isValidCallableFlowSite(value) {
+    if (!isRecord(value) || typeof value.kind !== 'string')
+        return false;
+    switch (value.kind) {
+        case 'seed':
+            return (isValidOperand(value.destination) &&
+                isBoundedString(value.targetName) &&
+                isOptionalBoundedString(value.targetQualifiedName) &&
+                isValidRange(value.targetRange) &&
+                isValidExpectedSignature(value.expectedSignature));
+        case 'copy':
+        case 'alias':
+        case 'address':
+            return isValidOperand(value.source) && isValidOperand(value.destination);
+        case 'store':
+            return isValidOperand(value.source) && isValidOperand(value.pointer);
+        case 'load':
+            return isValidOperand(value.pointer) && isValidOperand(value.destination);
+        case 'formal':
+            return (isBoundedString(value.ownerName) &&
+                isValidRange(value.ownerRange) &&
+                isSafeIndex(value.parameterIndex) &&
+                isValidOperand(value.binding) &&
+                (value.passingMode === 'value' ||
+                    value.passingMode === 'reference' ||
+                    value.passingMode === 'pointer') &&
+                isValidExpectedSignature(value.expectedSignature));
+        case 'argument':
+            return (isValidRange(value.callSite) &&
+                isSafeIndex(value.parameterIndex) &&
+                isValidOperand(value.source) &&
+                isOptionalBoundedString(value.directCalleeName));
+        case 'invoke':
+            return (isValidRange(value.callSite) &&
+                isBoundedString(value.inScope) &&
+                isValidOperand(value.callee) &&
+                (value.receiver === undefined || isValidOperand(value.receiver)) &&
+                (value.invocationKind === 'indirect' ||
+                    value.invocationKind === 'member-pointer' ||
+                    value.invocationKind === 'callable-object') &&
+                (value.arity === undefined || isSafeIndex(value.arity)));
+        default:
+            return false;
+    }
+}
+function isValidOperand(value) {
+    if (!isRecord(value))
+        return false;
+    return (isBoundedString(value.name) &&
+        isBoundedString(value.inScope) &&
+        isValidRange(value.atRange) &&
+        Number.isInteger(value.indirection) &&
+        value.indirection >= 0 &&
+        value.indirection <= MAX_CALLABLE_FLOW_INDIRECTION &&
+        typeof value.addressOf === 'boolean' &&
+        (value.expressionKind === undefined ||
+            value.expressionKind === 'binding' ||
+            value.expressionKind === 'callable-designator' ||
+            value.expressionKind === 'bound-member' ||
+            value.expressionKind === 'anonymous-callable') &&
+        isOptionalBoundedString(value.qualifiedName));
+}
+function isValidExpectedSignature(value) {
+    if (value === undefined)
+        return true;
+    if (!isRecord(value))
+        return false;
+    return ((value.parameterCount === undefined || isSafeIndex(value.parameterCount)) &&
+        isValidBoundedStringArray(value.parameterTypes) &&
+        (value.parameterTypeClasses === undefined ||
+            (Array.isArray(value.parameterTypeClasses) &&
+                value.parameterTypeClasses.length <= MAX_CALLABLE_FLOW_PARAMETERS &&
+                value.parameterTypeClasses.every(isRecord))) &&
+        (value.isConst === undefined || typeof value.isConst === 'boolean'));
+}
+function isValidBoundedStringArray(value) {
+    return (value === undefined ||
+        (Array.isArray(value) &&
+            value.length <= MAX_CALLABLE_FLOW_PARAMETERS &&
+            // '' is a legitimate entry meaning "unknown type" — the same convention
+            // as ReferenceSite.argumentTypes. C++ emits it for cv-qualifier-only or
+            // ERROR-recovered parameter types, so requiring non-empty here rejected
+            // real extractor output (#2522 review).
+            value.every((entry) => typeof entry === 'string' && entry.length <= MAX_CALLABLE_FLOW_NAME_LENGTH)));
+}
+function isValidRange(value) {
+    if (!isRecord(value))
+        return false;
+    return (isNonNegativeInteger(value.startLine) &&
+        isNonNegativeInteger(value.startCol) &&
+        isNonNegativeInteger(value.endLine) &&
+        isNonNegativeInteger(value.endCol));
+}
+function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function isBoundedString(value) {
+    return (typeof value === 'string' && value.length > 0 && value.length <= MAX_CALLABLE_FLOW_NAME_LENGTH);
+}
+function isOptionalBoundedString(value) {
+    return value === undefined || isBoundedString(value);
+}
+function isNonNegativeInteger(value) {
+    return Number.isSafeInteger(value) && value >= 0;
+}
+function isSafeIndex(value) {
+    return isNonNegativeInteger(value) && value <= MAX_CALLABLE_FLOW_PARAMETERS;
+}
+/** Durable store dir — a sibling of `parsedfile-store/`, NEVER cleared per run. */
+const getDurableParsedFileDir = (storagePath) => node_path_1.default.join(storagePath, DURABLE_DIRNAME);
+exports.getDurableParsedFileDir = getDurableParsedFileDir;
+const durableChunkDir = (durableDir, chunkHash) => node_path_1.default.join(durableDir, chunkHash);
+/**
+ * Start a fresh durable generation for one content-addressed parse chunk.
+ * The main thread calls this once before dispatching a cache miss, before any
+ * worker can write that chunk. Recreating the directory immediately keeps the
+ * worker-side mkdir memoization valid while preventing old worker shard names
+ * from accumulating across analyses.
+ */
+const prepareDurableParsedFileChunk = async (durableDir, chunkHash) => {
+    const dir = durableChunkDir(durableDir, chunkHash);
+    await node_fs_1.promises.rm(dir, { recursive: true, force: true });
+    await node_fs_1.promises.mkdir(dir, { recursive: true });
+};
+exports.prepareDurableParsedFileChunk = prepareDurableParsedFileChunk;
+// Per-process set of durable chunk subdirs already `mkdir`ed (mirrors
+// `createdStoreDirs`) so the worker doesn't `mkdirSync` on every shard.
+const createdDurableDirs = new Set();
+/**
+ * Synchronous durable-shard writer for use INSIDE a parse worker, alongside
+ * {@link persistParsedFileShardSync}. Writes the SAME bytes to a content-addressed
+ * durable location keyed by the parse chunk hash so a future warm hit can reuse
+ * them. `chunkHash`+`threadId`+`shardSeq` is collision-free across the
+ * N-shards-per-chunk fan-out and across worker-death retries — the same
+ * uniqueness that makes the run-scoped `w<tid>-<seq>` name safe, prefixed by
+ * content. No-op for an empty chunk.
+ */
+const persistDurableParsedFileShardSync = (durableDir, chunkHash, threadId, shardSeq, parsedFiles) => {
+    const payload = serializeParsedFileShard(parsedFiles);
+    if (payload === null)
+        return;
+    const dir = durableChunkDir(durableDir, chunkHash);
+    if (!createdDurableDirs.has(dir)) {
+        (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+        createdDurableDirs.add(dir);
+    }
+    (0, node_fs_1.writeFileSync)(node_path_1.default.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.json`), payload, 'utf-8');
+};
+exports.persistDurableParsedFileShardSync = persistDurableParsedFileShardSync;
+/**
+ * Restore a cached chunk's durable shards into the run-scoped store on a warm
+ * hit. A verbatim byte copy (no parse, no re-serialize), so the restored
+ * ParsedFiles are byte-identical to a cold run and `loadParsedFilesForPaths`
+ * (which keys on `filePath`, not shard name) gives scope-resolution full
+ * coverage. The durable shard names already carry the chunk hash, so they never
+ * collide with the worker's run-scoped `w<tid>-<seq>` shards. Returns the number
+ * of shards restored (0 ⇒ no durable coverage for this chunk; caller treats it
+ * as a miss).
+ */
+const restoreDurableParsedFileShard = async (durableDir, runStoragePath, chunkHash) => {
+    const src = durableChunkDir(durableDir, chunkHash);
+    let shards;
+    try {
+        shards = (await node_fs_1.promises.readdir(src)).filter((f) => f.endsWith('.json'));
+    }
+    catch {
+        return 0; // no durable shards for this chunk
+    }
+    if (shards.length === 0)
+        return 0;
+    const dst = (0, exports.getParsedFileStoreDir)(runStoragePath);
+    await node_fs_1.promises.mkdir(dst, { recursive: true });
+    for (const name of shards) {
+        await node_fs_1.promises.copyFile(node_path_1.default.join(src, name), node_path_1.default.join(dst, name));
+    }
+    return shards.length;
+};
+exports.restoreDurableParsedFileShard = restoreDurableParsedFileShard;
+/**
+ * Read the durable index and return the set of chunk hashes it vouches for,
+ * gated on `expectedVersion` (`PARSE_CACHE_VERSION`). A version mismatch or a
+ * missing/corrupt index returns the empty set — the caller then treats every
+ * chunk as a durable miss and re-dispatches workers (NEVER the main-thread
+ * `extractParsedFile` fallback), which rewrites the durable store under the new
+ * version. Mirrors `loadParseCache`'s version-invalidation contract.
+ */
+const loadDurableParsedFileIndex = async (durableDir, expectedVersion) => {
+    try {
+        const raw = await node_fs_1.promises.readFile(node_path_1.default.join(durableDir, DURABLE_INDEX_FILENAME), 'utf-8');
+        const idx = JSON.parse(raw);
+        if (idx?.version !== expectedVersion || !Array.isArray(idx.keys))
+            return new Set();
+        return new Set(idx.keys);
+    }
+    catch {
+        return new Set();
+    }
+};
+exports.loadDurableParsedFileIndex = loadDurableParsedFileIndex;
+/**
+ * Prune the durable store to `keepKeys` and rewrite its index. `keepKeys` must
+ * be the parse cache's surviving on-disk keys (so the two stores stay coherent:
+ * a chunk is "cached" iff BOTH its parse-cache shard and its durable shards
+ * exist; a quarantined chunk — no parse-cache shard — drops its durable subdir
+ * here and re-dispatches next run). Only subdirs with ≥1 shard are indexed
+ * (mirrors `saveParseCache`'s written-keys discipline — never vouch for a chunk
+ * hash with no backing shard). The index write is tmp+rename atomic.
+ */
+const pruneAndSaveDurableParsedFileStore = async (durableDir, version, keepKeys) => {
+    let entries;
+    try {
+        entries = await node_fs_1.promises.readdir(durableDir);
+    }
+    catch {
+        return; // nothing written this run
+    }
+    const survivors = [];
+    for (const name of entries) {
+        if (name === DURABLE_INDEX_FILENAME)
+            continue;
+        const full = node_path_1.default.join(durableDir, name);
+        if (keepKeys.has(name)) {
+            try {
+                const shards = (await node_fs_1.promises.readdir(full)).filter((f) => f.endsWith('.json'));
+                if (shards.length > 0) {
+                    survivors.push(name);
+                    continue;
+                }
+            }
+            catch {
+                /* not a readable dir → drop below */
+            }
+        }
+        await node_fs_1.promises.rm(full, { recursive: true, force: true });
+    }
+    const idx = { version, keys: survivors };
+    const tmp = node_path_1.default.join(durableDir, `${DURABLE_INDEX_FILENAME}.tmp`);
+    await node_fs_1.promises.mkdir(durableDir, { recursive: true });
+    await node_fs_1.promises.writeFile(tmp, JSON.stringify(idx), 'utf-8');
+    await node_fs_1.promises.rename(tmp, node_path_1.default.join(durableDir, DURABLE_INDEX_FILENAME));
+};
+exports.pruneAndSaveDurableParsedFileStore = pruneAndSaveDurableParsedFileStore;