gitnexus 1.6.12-rc.39 → 1.6.12-rc.40

package/dist/core/ingestion/pipeline-phases/parse-impl.js

@@ -16,8 +16,8 @@
  */
 import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
 import { mergeChunkResults, dispatchChunkParseRound } from '../parsing-processor.js';
-import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
-import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, } from '../../../storage/parsedfile-store.js';
+import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, markParseCacheChunkStale, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
+import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, durableChunkHasStaleShards, } from '../../../storage/parsedfile-store.js';
 import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
 import { processRoutesFromExtracted, resolveRouteHandlerSymbols, buildExportedTypeMapFromGraph, } from '../call-processor.js';
 import { createSemanticModel } from '../model/index.js';
@@ -669,8 +669,12 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
          * Chunk hashes whose durable ParsedFile directory could not be reset. The
          * old generation's shards are still on disk, so a warm hit would union
          * stale shards with the new ones. Treated exactly like a quarantined chunk:
-         * skip the parse-cache write so the next run re-dispatches into a clean
-         * directory rather than trusting a generation we could not clear.
+         * skip the parse-cache write AND, when shards from that generation are
+         * still on disk, retire the hash (#3204): the old `.v8` is not carried
+         * forward and the directory is dropped from the durable index, so the next
+         * run re-dispatches. Kept as its own set rather than
+         * read back off `staleKeys`, which is a superset — it is what selects the
+         * warn below over the quarantine branch's dev-only log.
          */
         const durablePrepareFailures = new Set();
         const roundByteBudget = resolveParseRoundByteBudget(options);
@@ -809,7 +813,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
             // Persist raw results for this chunk hash (skipping when any chunk file
             // was worker-quarantined, so the narrower rawResults isn't cached under
             // the full-chunk key — see the original inline note / U20.U2).
-            if (parseCache && p.chunkHash && rawResults.length > 0) {
+            // `rawResults.length > 0` guards the WRITE only. A quarantined chunk
+            // often returns nothing at all (the worker died on it), and that chunk
+            // still has to be retired — otherwise its pre-existing `.v8` is copied
+            // forward at save time (#3204).
+            if (parseCache && p.chunkHash) {
                 const quarantineSet = new Set(workerPool?.getQuarantinedPaths?.() ?? []);
                 const chunkHadQuarantine = p.chunkFiles.some((f) => quarantineSet.has(f.path));
                 const durableGenerationStale = durablePrepareFailures.has(p.chunkHash);
@@ -818,6 +826,10 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                         'so its shards may be stale; next run will re-dispatch it');
                 }
                 else if (chunkHadQuarantine) {
+                    // This chunk's durable directory now holds only this run's NARROWER
+                    // shards, so a warm hit would replay the full-coverage `.v8` over
+                    // partial ParsedFiles (#3204).
+                    markParseCacheChunkStale(parseCache, p.chunkHash);
                     if (isDev) {
                         const quarantinedInChunk = p.chunkFiles.filter((f) => quarantineSet.has(f.path)).length;
                         logger.info(`📦 parse-cache SKIP: chunk ${p.chunkIdx + 1}/${numChunks} ` +
@@ -825,7 +837,7 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                             `next run will rediscover (${p.chunkHash.slice(0, 8)})`);
                     }
                 }
-                else {
+                else if (rawResults.length > 0) {
                     await persistParseCacheChunk(parseCache, p.chunkHash, rawResults);
                     if (isDev) {
                         logger.info(`📦 parse-cache MISS+store: chunk ${p.chunkIdx + 1}/${numChunks} (${p.chunkFiles.length} files, ${p.chunkHash.slice(0, 8)})`);
@@ -871,6 +883,20 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                     // directory on write, so at worst the old generation lingers.
                     // Caught per chunk so one failure cannot abort the others.
                     durablePrepareFailures.add(miss.chunkHash);
+                    // Retire ONLY when a generation nobody cleared is still on disk.
+                    // `prepareDurableParsedFileChunk` is rm-then-mkdir: an rm failure
+                    // leaves the old shards to be unioned into a warm hit, but an rm
+                    // that succeeded before a failing mkdir leaves nothing — the
+                    // workers recreate the directory and write a clean generation, so
+                    // retiring there would discard a good `.v8` for no safety gain
+                    // (and a correlated burst would discard the whole shared cache).
+                    // Retire here rather than at finalize: that branch sits behind
+                    // `rawResults.length > 0`, so a chunk whose worker round returns
+                    // nothing would keep its stale entry.
+                    if (parseCache &&
+                        (await durableChunkHasStaleShards(durableParsedFileDir, miss.chunkHash))) {
+                        markParseCacheChunkStale(parseCache, miss.chunkHash);
+                    }
                     logger.warn({ err, chunkHash: miss.chunkHash.slice(0, 8) }, 'parsedfile-cache: could not reset durable chunk generation; ' +
                         'continuing without caching this chunk');
                 }

package/dist/core/run-analyze.js

@@ -3437,9 +3437,11 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
             // Prune the durable ParsedFile store to EXACTLY the parse cache's
             // surviving keys (#2038 warm-cache coverage), so the two content-addressed
             // stores stay coherent: a chunk is "cached" iff both its parse-cache shard
-            // and its durable shards exist. A quarantined chunk (in usedKeys but with
-            // no parse-cache shard) drops its durable subdir here and re-dispatches
-            // next run. Same try/catch — a durable-store write failure must never
+            // and its durable shards exist. A retired chunk — worker-quarantined, or
+            // one whose failed durable reset left an uncleared generation behind
+            // (#3204) — is filtered out of `savedKeys`, so it drops out of the
+            // durable index here and re-dispatches next run. Same try/catch — a
+            // durable-store write must never
             // break an otherwise successful run (next run treats it as a miss).
             await mergeStagedDurableParsedFileStore(storagePath, parseCache.storagePath ?? storagePath, PARSE_CACHE_VERSION, new Set(savedKeys));
         }

package/dist/storage/parse-cache.d.ts

@@ -68,6 +68,16 @@ export interface ParseCache {
      * Transient — never serialized to disk.
      */
     usedKeys: Set<string>;
+    /**
+     * Hashes this run decided it cannot vouch for — its durable generation could
+     * not be reset, or its chunk was worker-quarantined (#3204). `saveParseCache`
+     * refuses them, so neither a pre-existing `.v8` nor the chunk's durable
+     * directory survives into the next run. Kept separate from `usedKeys`
+     * because the orchestrator re-adds keys to that set after the parse phase
+     * (#2106 sibling fold), which would undo a deletion.
+     * Transient — never serialized to disk.
+     */
+    staleKeys?: Set<string>;
     /**
      * When set, chunk payloads are loaded from / flushed to sharded files on
      * demand instead of retaining every chunk in `entries` for the whole run
@@ -129,6 +139,18 @@ export declare const forgetCreatedParseCacheDir: (storagePath: string) => void;
  * run. Falls back to `cache.entries` when `storagePath` is unset (unit tests).
  */
 export declare const persistParseCacheChunk: (cache: ParseCache, chunkHash: string, chunkResults: readonly ParseWorkerResult[]) => Promise<void>;
+/**
+ * Retire a chunk this run cannot vouch for — its durable ParsedFile generation
+ * could not be reset, or its chunk was worker-quarantined (#3204).
+ *
+ * `saveParseCache` refuses a stale key, so no pre-existing `.v8` is copied
+ * forward and the durable store — pruned to exactly the keys that save
+ * returns — drops the chunk in the same step. The two deletes matter because
+ * `loadParseCacheChunk` reads `entries` and `onDiskKeys` and does NOT consult
+ * `staleKeys`: without them a second lookup of the same hash inside this run
+ * would still serve the retired shard.
+ */
+export declare const markParseCacheChunkStale: (cache: ParseCache, chunkHash: string) => void;
 /**
  * Load the parse cache. Returns an empty cache on any failure (missing
  * file, corrupt JSON, version mismatch). Never throws on a normal load.

package/dist/storage/parse-cache.js

@@ -1017,6 +1017,23 @@ export const persistParseCacheChunk = async (cache, chunkHash, chunkResults) =>
     }
     cache.entries.set(chunkHash, slim);
 };
+/**
+ * Retire a chunk this run cannot vouch for — its durable ParsedFile generation
+ * could not be reset, or its chunk was worker-quarantined (#3204).
+ *
+ * `saveParseCache` refuses a stale key, so no pre-existing `.v8` is copied
+ * forward and the durable store — pruned to exactly the keys that save
+ * returns — drops the chunk in the same step. The two deletes matter because
+ * `loadParseCacheChunk` reads `entries` and `onDiskKeys` and does NOT consult
+ * `staleKeys`: without them a second lookup of the same hash inside this run
+ * would still serve the retired shard.
+ */
+export const markParseCacheChunkStale = (cache, chunkHash) => {
+    cache.staleKeys ??= new Set();
+    cache.staleKeys.add(chunkHash);
+    cache.entries.delete(chunkHash);
+    cache.onDiskKeys?.delete(chunkHash);
+};
 const loadLegacyParseCache = async (storagePath) => {
     const cachePath = getLegacyCachePath(storagePath);
     try {
@@ -1096,7 +1113,14 @@ export const saveParseCache = async (storagePath, cache) => {
     const tmpDir = `${cacheDir}.tmp`;
     await fs.rm(tmpDir, { recursive: true, force: true });
     await fs.mkdir(tmpDir, { recursive: true });
-    const keys = [...cache.usedKeys].filter(isValidChunkCacheKey).sort();
+    // A stale key is dropped here rather than at the failure site: the
+    // orchestrator folds sibling-branch keys back into `usedKeys` after the parse
+    // phase (#2106), so this is the last point that sees the final key set. The
+    // exclusion also reaches the durable store, which prunes to the keys this
+    // function returns — both stores drop the chunk together (#3204).
+    const keys = [...cache.usedKeys]
+        .filter((key) => isValidChunkCacheKey(key) && !cache.staleKeys?.has(key))
+        .sort();
     // Track hashes whose shard was actually written/copied this save. A hash can
     // be in `usedKeys` without a backing shard — its in-memory serialize threw, or
     // its on-disk copy failed/was-absent (e.g. a worker-quarantined chunk added to

package/dist/storage/parsedfile-store.d.ts

@@ -103,6 +103,18 @@ export declare const getDurableParsedFileDir: (storagePath: string) => string;
  * from accumulating across analyses.
  */
 export declare const prepareDurableParsedFileChunk: (durableDir: string, chunkHash: string) => Promise<void>;
+/**
+ * Does this chunk still hold shards from a generation nobody cleared?
+ *
+ * Asked only after {@link prepareDurableParsedFileChunk} rejected, to tell its
+ * two failure modes apart (#3204). The `rm` failing leaves the previous
+ * generation in place, and a warm hit would union it with whatever this run's
+ * workers write — that chunk must be retired. The `rm` succeeding and the
+ * `mkdir` then failing leaves NO directory: the workers recreate it and write
+ * a clean generation, so retiring would throw away a good cache entry for
+ * nothing. Absent or empty ⇒ nothing to distrust.
+ */
+export declare const durableChunkHasStaleShards: (durableDir: string, chunkHash: string) => Promise<boolean>;
 export declare const persistDurableParsedFileShardSync: (durableDir: string, chunkHash: string, threadId: number, shardSeq: number, parsedFiles: readonly ParsedFile[]) => boolean;
 /**
  * Validate and snapshot one durable chunk into the run store. Every envelope
@@ -116,8 +128,9 @@ export declare const loadDurableParsedFileIndex: (durableDir: string, expectedVe
  * Prune the durable store to `keepKeys` and rewrite its index. `keepKeys` must
  * be the parse cache's surviving on-disk keys (so the two stores stay coherent:
  * a chunk is "cached" iff BOTH its parse-cache shard and its durable shards
- * exist; a quarantined chunk — no parse-cache shard — drops its durable subdir
- * here and re-dispatches next run). Only chunks whose envelopes all validate
+ * exist; a chunk retired by `markParseCacheChunkStale` — worker-quarantined, or
+ * holding a durable generation that could not be reset — is absent from
+ * `keepKeys`, so it drops its durable subdir here and re-dispatches next run). Only chunks whose envelopes all validate
  * are indexed, together with their exact persisted path coverage (never vouch
  * for a missing/corrupt shard). The index write is tmp+rename atomic.
  */

package/dist/storage/parsedfile-store.js

@@ -512,6 +512,18 @@ export const prepareDurableParsedFileChunk = async (durableDir, chunkHash) => {
     await fs.rm(dir, { recursive: true, force: true });
     await fs.mkdir(dir, { recursive: true });
 };
+/**
+ * Does this chunk still hold shards from a generation nobody cleared?
+ *
+ * Asked only after {@link prepareDurableParsedFileChunk} rejected, to tell its
+ * two failure modes apart (#3204). The `rm` failing leaves the previous
+ * generation in place, and a warm hit would union it with whatever this run's
+ * workers write — that chunk must be retired. The `rm` succeeding and the
+ * `mkdir` then failing leaves NO directory: the workers recreate it and write
+ * a clean generation, so retiring would throw away a good cache entry for
+ * nothing. Absent or empty ⇒ nothing to distrust.
+ */
+export const durableChunkHasStaleShards = async (durableDir, chunkHash) => (await listV8Shards(durableChunkDir(durableDir, chunkHash))).length > 0;
 /**
  * Synchronous durable-shard writer for use INSIDE a parse worker, alongside
  * {@link persistParsedFileShardSync}. Writes the SAME bytes to a content-addressed
@@ -606,8 +618,9 @@ export const loadDurableParsedFileIndex = async (durableDir, expectedVersion) =>
  * Prune the durable store to `keepKeys` and rewrite its index. `keepKeys` must
  * be the parse cache's surviving on-disk keys (so the two stores stay coherent:
  * a chunk is "cached" iff BOTH its parse-cache shard and its durable shards
- * exist; a quarantined chunk — no parse-cache shard — drops its durable subdir
- * here and re-dispatches next run). Only chunks whose envelopes all validate
+ * exist; a chunk retired by `markParseCacheChunkStale` — worker-quarantined, or
+ * holding a durable generation that could not be reset — is absent from
+ * `keepKeys`, so it drops its durable subdir here and re-dispatches next run). Only chunks whose envelopes all validate
  * are indexed, together with their exact persisted path coverage (never vouch
  * for a missing/corrupt shard). The index write is tmp+rename atomic.
  */
@@ -620,6 +633,8 @@ export const pruneAndSaveDurableParsedFileStore = async (durableDir, version, ke
         return; // nothing written this run
     }
     const survivors = {};
+    const undeletable = [];
+    let firstRemoveError;
     for (const name of entries) {
         if (name === DURABLE_INDEX_FILENAME)
             continue;
@@ -649,7 +664,27 @@ export const pruneAndSaveDurableParsedFileStore = async (durableDir, version, ke
                 /* not a readable dir → drop below */
             }
         }
-        await fs.rm(full, { recursive: true, force: true });
+        // The causes that break `prepareDurableParsedFileChunk` — permissions, a
+        // locked file, a read-only mount — break this rm too (#3204). Dropping the
+        // entry from the index is what makes the chunk unreachable; losing the
+        // directory is a cleanup bonus. Never let one of them abort the loop and
+        // cost every remaining chunk its index entry.
+        try {
+            await fs.rm(full, { recursive: true, force: true });
+        }
+        catch (err) {
+            // `name` reached the drop branch precisely because it is not a live
+            // chunk key, so it may be any stray directory — report it as an entry.
+            undeletable.push(name);
+            firstRemoveError ??= err;
+        }
+    }
+    if (undeletable.length > 0) {
+        // One line per RUN, not per directory: a store-wide cause (read-only mount,
+        // wrong ownership) hits every non-survivor, and thousands of warns would
+        // bury the message that matters.
+        logger.warn({ err: firstRemoveError, count: undeletable.length, firstEntry: undeletable[0] }, 'parsedfile-cache: could not remove pruned durable chunk directories; ' +
+            'they are excluded from the index and will be re-attempted next run');
     }
     const idx = { version, entries: survivors };
     const tmp = path.join(durableDir, `${DURABLE_INDEX_FILENAME}.tmp`);
@@ -684,7 +719,18 @@ export const mergeStagedDurableParsedFileStore = async (liveStoragePath, stagedS
             continue;
         const from = path.join(stagedDir, name);
         const to = path.join(liveDir, name);
-        await replaceDurableChunkDir(from, to);
+        // Same reasoning as the prune's per-entry guard, on the loop that runs
+        // BEFORE it (#3204): this overlay targets the same live chunk directories,
+        // so the causes that break a reset break a replacement too. Letting one
+        // throw here would skip the prune entirely — the durable index would not be
+        // rewritten this run, and a retired chunk would keep its directory.
+        try {
+            await replaceDurableChunkDir(from, to);
+        }
+        catch (err) {
+            logger.warn({ err, entry: name }, 'parsedfile-cache: could not publish a staged durable chunk; ' +
+                'it stays uncached and will re-dispatch next run');
+        }
     }
     await pruneAndSaveDurableParsedFileStore(liveDir, version, keepKeys);
 };
@@ -698,6 +744,12 @@ const replaceDurableChunkDir = async (from, to) => {
         /* dest exists, or the rename is cross-device */
     }
     const backup = `${to}.replacing`;
+    // Deliberately NOT best-effort. If a non-empty backup survives, the
+    // `fs.rename(to, backup)` below cannot overwrite it and is swallowed as
+    // "dest was missing", so the `fs.cp` fallback would merge the staged
+    // generation INTO the live directory — manufacturing exactly the old+new
+    // union this fix exists to prevent. Let it throw; the caller's per-entry
+    // guard keeps one such chunk from costing the others their prune.
     await fs.rm(backup, { recursive: true, force: true });
     let backedUp = false;
     try {
@@ -724,6 +776,8 @@ const replaceDurableChunkDir = async (from, to) => {
         throw err;
     }
     if (backedUp) {
-        await fs.rm(backup, { recursive: true, force: true });
+        // The new generation is already in place; an undeletable backup is litter,
+        // not a failure. The next prune re-attempts it.
+        await fs.rm(backup, { recursive: true, force: true }).catch(() => { });
     }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.12-rc.39",
+  "version": "1.6.12-rc.40",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",

package/scripts/cross-platform-tests.ts

@@ -290,6 +290,11 @@ const NATIVE_ADDON_SMOKE = [
 // Filesystem behavior tests — exercise operations that vary across
 // platforms (CRLF, symlinks, permissions, temp dirs)
 const FILESYSTEM = [
+  // The durable ParsedFile store's prune tolerates a chunk directory it cannot
+  // delete (#3204). The failures that motivate it — held handles, read-only
+  // mounts — are Windows- and macOS-flavored, and the permission-based case
+  // skips itself where chmod cannot block a delete, so run it everywhere.
+  'test/unit/parsedfile-store.test.ts',
   'test/integration/filesystem-walker.test.ts',
   'test/integration/watch-filesystem.test.ts',
   'test/integration/markdown-processor-crlf.test.ts',