gitnexus 1.6.12-rc.3 → 1.6.12-rc.4

package/dist/core/ingestion/parsing-processor.d.ts

@@ -47,12 +47,12 @@ export interface WorkerExtractedData {
  */
 export declare const mergeChunkResults: (graph: KnowledgeGraph, symbolTable: SymbolTableWriter, chunkResults: readonly ParseWorkerResult[], exportedTypeMap?: ExportedTypeMap) => WorkerExtractedData;
 /**
- * Dispatch a chunk's files to the worker pool and return the RAW per-worker
- * results, WITHOUT merging them into the graph. Split out from
- * {@link processParsing} so the parse loop can overlap one chunk's
- * merge (main-thread, via {@link mergeChunkResults}) with the NEXT chunk's
- * worker parse — the merge is the only remaining serial main-thread step once
- * ParsedFile serialization moved into the workers (#worker-idle pipelining).
+ * Dispatch ONE chunk's files to the worker pool and return the RAW per-worker
+ * results, WITHOUT merging them into the graph. A thin single-group wrapper
+ * over {@link dispatchChunkParseRound}, used by {@link processParsing}'s
+ * one-shot path. The chunk-to-chunk overlap this once described now lives in
+ * `parse-impl.ts` at ROUND granularity (`startRound` / `drainRound` /
+ * `closeRound`), which batches several chunks into one dispatch.
  * Returns `[]` for an all-unparseable chunk (the caller merges `[]` → empty).
  */
 export declare const dispatchChunkParse: (files: {
@@ -67,6 +67,24 @@ outRawResults?: ParseWorkerResult[],
  * (#2038). `undefined` ⇒ no durable write (tests / no-cache path).
  */
 chunkHash?: string) => Promise<ParseWorkerResult[]>;
+/**
+ * Dispatch SEVERAL parse-cache chunks as one pool round and return their raw
+ * results, one array per input group in input order.
+ *
+ * `WorkerPool.dispatch` is a barrier, so one round-trip per chunk leaves most
+ * slots idle whenever a chunk is smaller than the pool — which stable
+ * `(language, hash(path) % 128)` packs usually are. Batching chunks into one
+ * `dispatchGroups` call removes those barriers; jobs are still cut at chunk
+ * boundaries, so every result stays attributable to the chunk whose cache key
+ * owns it.
+ */
+export declare const dispatchChunkParseRound: (groups: ReadonlyArray<{
+    items: {
+        path: string;
+        content: string;
+    }[];
+    chunkHash?: string;
+}>, workerPool: WorkerPool, onFileProgress?: FileProgressCallback) => Promise<ParseWorkerResult[][]>;
 export declare const processParsing: (graph: KnowledgeGraph, files: {
     path: string;
     content: string;

package/dist/core/ingestion/parsing-processor.js

@@ -147,12 +147,12 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
     };
 };
 /**
- * Dispatch a chunk's files to the worker pool and return the RAW per-worker
- * results, WITHOUT merging them into the graph. Split out from
- * {@link processParsing} so the parse loop can overlap one chunk's
- * merge (main-thread, via {@link mergeChunkResults}) with the NEXT chunk's
- * worker parse — the merge is the only remaining serial main-thread step once
- * ParsedFile serialization moved into the workers (#worker-idle pipelining).
+ * Dispatch ONE chunk's files to the worker pool and return the RAW per-worker
+ * results, WITHOUT merging them into the graph. A thin single-group wrapper
+ * over {@link dispatchChunkParseRound}, used by {@link processParsing}'s
+ * one-shot path. The chunk-to-chunk overlap this once described now lives in
+ * `parse-impl.ts` at ROUND granularity (`startRound` / `drainRound` /
+ * `closeRound`), which batches several chunks into one dispatch.
  * Returns `[]` for an all-unparseable chunk (the caller merges `[]` → empty).
  */
 export const dispatchChunkParse = async (files, workerPool, onFileProgress, 
@@ -164,23 +164,42 @@ outRawResults,
  * (#2038). `undefined` ⇒ no durable write (tests / no-cache path).
  */
 chunkHash) => {
-    const parseableFiles = [];
-    for (const file of files) {
-        const lang = getLanguageFromFilename(file.path);
-        if (lang)
-            parseableFiles.push({ path: file.path, content: file.content });
-    }
-    if (parseableFiles.length === 0)
-        return [];
-    const total = files.length;
-    const chunkResults = await workerPool.dispatch(parseableFiles, (filesProcessed) => {
-        onFileProgress?.(Math.min(filesProcessed, total), total, 'Parsing...');
-    }, chunkHash);
+    const [chunkResults = []] = await dispatchChunkParseRound([{ items: files, chunkHash }], workerPool, onFileProgress);
     // Capture raw results for the incremental parse cache before merging.
     if (outRawResults) {
         for (const r of chunkResults)
             outRawResults.push(r);
     }
+    return chunkResults;
+};
+/**
+ * Dispatch SEVERAL parse-cache chunks as one pool round and return their raw
+ * results, one array per input group in input order.
+ *
+ * `WorkerPool.dispatch` is a barrier, so one round-trip per chunk leaves most
+ * slots idle whenever a chunk is smaller than the pool — which stable
+ * `(language, hash(path) % 128)` packs usually are. Batching chunks into one
+ * `dispatchGroups` call removes those barriers; jobs are still cut at chunk
+ * boundaries, so every result stays attributable to the chunk whose cache key
+ * owns it.
+ */
+export const dispatchChunkParseRound = async (groups, workerPool, onFileProgress) => {
+    const dispatchGroups = groups.map((group) => {
+        const items = [];
+        for (const file of group.items) {
+            const lang = getLanguageFromFilename(file.path);
+            if (lang)
+                items.push({ path: file.path, content: file.content });
+        }
+        return { items, chunkHash: group.chunkHash };
+    });
+    const total = groups.reduce((sum, group) => sum + group.items.length, 0);
+    if (dispatchGroups.every((group) => group.items.length === 0))
+        return groups.map(() => []);
+    const perGroup = await workerPool.dispatchGroups(dispatchGroups, (filesProcessed) => {
+        onFileProgress?.(Math.min(filesProcessed, total), total, 'Parsing...');
+    });
+    const chunkResults = perGroup.flat();
     // Skipped-language telemetry (worker output, independent of the merge).
     const skippedLanguages = new Map();
     for (const result of chunkResults) {
@@ -238,7 +257,7 @@ chunkHash) => {
         logger.warn(`  Sanitized ${skippedPaths.length} file(s) with non-serializable parse output: ${shown}${more}`);
     }
     onFileProgress?.(total, total, 'done');
-    return chunkResults;
+    return perGroup;
 };
 // ============================================================================
 // Public API

package/dist/core/ingestion/pipeline-phases/parse-impl.js

@@ -15,7 +15,7 @@
  * @module
  */
 import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
-import { mergeChunkResults, dispatchChunkParse } from '../parsing-processor.js';
+import { mergeChunkResults, dispatchChunkParseRound } from '../parsing-processor.js';
 import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
 import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, } from '../../../storage/parsedfile-store.js';
 import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
@@ -132,6 +132,28 @@ const CHUNK_BYTES_PER_WORKER = DEFAULT_CHUNK_BYTE_BUDGET;
 const TARGET_JOBS_PER_WORKER = 3;
 /** Floor for a derived sub-batch so jobs don't shrink to per-file IPC churn. */
 const MIN_SUB_BATCH_BYTES = 256 * 1024;
+/**
+ * Source bytes of cache-missing chunks allowed in flight in one pool round.
+ *
+ * A `dispatch` is a barrier, so one round-trip per cache pack leaves most slots
+ * idle: packs are keyed by `(language, hash(path) % 128)` and routinely land far
+ * under {@link DEFAULT_CHUNK_BYTE_BUDGET} (this repo: 1285 packs where the byte
+ * budget alone needs 16, 549 of them holding a single file). Rounds batch packs
+ * into one `dispatchGroups` call without touching pack identity.
+ *
+ * This is the in-flight cap, the same role Piscina's `maxQueue` plays: bigger
+ * rounds remove more barriers but hold more file content and more un-merged
+ * worker output on the main thread at once. Defaulting to one chunk budget
+ * keeps in-flight source bytes at the magnitude the loop already prefetched
+ * (`parseChunkConcurrency`, 2 chunks ahead). Override via
+ * `GITNEXUS_PARSE_ROUND_BYTES`.
+ */
+function resolveParseRoundByteBudget(options) {
+    const env = Number(process.env.GITNEXUS_PARSE_ROUND_BYTES);
+    if (Number.isFinite(env) && env > 0)
+        return env;
+    return resolveChunkByteBudget(options);
+}
 function resolveChunkByteBudget(options) {
     const opt = options?.chunkByteBudget;
     if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0)
@@ -596,13 +618,33 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         // the env can't change mid-run.
         const verboseThroughputLog = isDev || isVerboseIngestionEnabled();
         const heapProbeEveryN = isDebugHeapEnabled() ? 25 : 0;
-        let pendingWorkerChunk = null;
+        const roundByteBudget = resolveParseRoundByteBudget(options);
+        let roundEntries = [];
+        let roundMissBytes = 0;
+        /**
+         * Bytes an open round is HOLDING, counting hits as well as misses.
+         *
+         * `roundMissBytes` alone bounds only what the workers are asked to do, so a
+         * warm run — where nothing misses — would never reach the close condition
+         * and would buffer every chunk's cached output until the tail drain. That
+         * is the #2649 heap failure on a large repo. Closing on either cap keeps a
+         * hits-only run draining at the same cadence as a cold one; `startRound`
+         * already supports a round with no misses.
+         */
+        let roundBufferedBytes = 0;
+        /**
+         * Files QUEUED into rounds so far. `filesParsedSoFar` only advances when a
+         * round drains, so it is the right number for the throughput log but would
+         * pin a warm run's progress bar at the phase floor for the whole loop.
+         */
+        let queuedFilesSoFar = 0;
+        let pendingRound = null;
         // Apply one chunk's merged worker data: per-chunk aggregation into the
         // run-level accumulators + the throughput log. Shared by the cache-hit
         // (inline) and worker (deferred) paths. The `| null` guard is defensive —
         // every live caller passes real worker data now that sequential parsing
         // (which was the only path that passed null) is gone.
-        const applyChunkResults = async (chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs) => {
+        const applyChunkResults = async (chunkWorkerData, chunkIdx, fileCount, chunkStartMs) => {
             if (chunkWorkerData) {
                 for (const filePath of chunkWorkerData.scopeExtractionFailures) {
                     scopeExtractionFailures.add(filePath);
@@ -690,27 +732,27 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                         allORMQueries.push(item);
                 }
             }
-            filesParsedSoFar += chunkFiles.length;
+            filesParsedSoFar += fileCount;
             if (verboseThroughputLog && chunkStartMs !== null) {
                 const elapsedMs = Date.now() - chunkStartMs;
-                const filesPerSec = elapsedMs > 0 ? (chunkFiles.length * 1000) / elapsedMs : 0;
+                const filesPerSec = elapsedMs > 0 ? (fileCount * 1000) / elapsedMs : 0;
                 const stats = workerPool?.getStats?.();
                 const poolFrag = stats
                     ? ` pool: ${stats.activeSlots}/${stats.size} active, ` +
                         `${stats.quarantined} quarantined${stats.poolBroken ? ', BROKEN' : ''}`
                     : ' (cache replay)';
-                logger.info(`📊 chunk ${chunkIdx + 1}/${numChunks}: ${chunkFiles.length} files in ${elapsedMs}ms ` +
+                logger.info(`📊 chunk ${chunkIdx + 1}/${numChunks}: ${fileCount} files in ${elapsedMs}ms ` +
                     `(${filesPerSec.toFixed(1)} files/s)${poolFrag}`);
             }
         };
         // Merge + finalize a parked worker chunk: graph merge (the overlapped
         // main-thread step) → parse-cache write-guard → run-level aggregation.
-        const finalizeWorkerChunk = async (p) => {
-            const chunkWorkerData = mergeChunkResults(graph, symbolTable, p.rawResults, exportedTypeMap);
+        const finalizeWorkerChunk = async (p, rawResults) => {
+            const chunkWorkerData = mergeChunkResults(graph, symbolTable, rawResults, exportedTypeMap);
             // Persist raw results for this chunk hash (skipping when any chunk file
             // was worker-quarantined, so the narrower rawResults isn't cached under
             // the full-chunk key — see the original inline note / U20.U2).
-            if (parseCache && p.chunkHash && p.rawResults.length > 0) {
+            if (parseCache && p.chunkHash && rawResults.length > 0) {
                 const quarantineSet = new Set(workerPool?.getQuarantinedPaths?.() ?? []);
                 const chunkHadQuarantine = p.chunkFiles.some((f) => quarantineSet.has(f.path));
                 if (chunkHadQuarantine) {
@@ -722,13 +764,150 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                     }
                 }
                 else {
-                    await persistParseCacheChunk(parseCache, p.chunkHash, p.rawResults);
+                    await persistParseCacheChunk(parseCache, p.chunkHash, rawResults);
                     if (isDev) {
                         logger.info(`📦 parse-cache MISS+store: chunk ${p.chunkIdx + 1}/${numChunks} (${p.chunkFiles.length} files, ${p.chunkHash.slice(0, 8)})`);
                     }
                 }
             }
-            await applyChunkResults(chunkWorkerData, p.chunkIdx, p.chunkFiles, p.chunkStartMs);
+            await applyChunkResults(chunkWorkerData, p.chunkIdx, p.chunkFiles.length, p.chunkStartMs);
+        };
+        /**
+         * Dispatch a round's cache misses as ONE pool round. Returns the parked
+         * round; the caller drains it after starting the next one so the workers
+         * parse round N+1 while the main thread merges round N (the same overlap
+         * the per-chunk loop had, at round granularity).
+         */
+        const startRound = async (entries) => {
+            if (entries.length === 0)
+                return null;
+            const misses = entries.filter((entry) => entry.kind === 'miss');
+            if (misses.length === 0) {
+                return { entries, results: Promise.resolve([]) };
+            }
+            for (const miss of misses) {
+                if (durableParsedFileDir !== undefined && miss.chunkHash !== null) {
+                    try {
+                        await prepareDurableParsedFileChunk(durableParsedFileDir, miss.chunkHash);
+                    }
+                    catch (err) {
+                        // The durable store is an optimization — degrade like the restore
+                        // path does instead of failing the analyze. Workers recreate the
+                        // directory on write, so at worst the old generation lingers.
+                        logger.warn({ err, chunkHash: miss.chunkHash.slice(0, 8) }, 'parsedfile-cache: could not reset durable chunk generation; continuing');
+                    }
+                }
+            }
+            const roundFiles = misses.reduce((sum, miss) => sum + miss.chunkFiles.length, 0);
+            const firstIdx = misses[0].chunkIdx;
+            const lastIdx = misses[misses.length - 1].chunkIdx;
+            const progressForRound = (current, _total, filePath) => {
+                // Rounds queued before this one are already counted in
+                // `queuedFilesSoFar`; `current` is this round's own worker progress.
+                const globalCurrent = queuedFilesSoFar - roundFiles + current;
+                // Parse phase covers 20-70 (M2). Deferred extraction handles 70-95.
+                const parsingProgress = 20 + (globalCurrent / totalParseable) * 50;
+                onProgress({
+                    phase: 'parsing',
+                    percent: Math.round(parsingProgress),
+                    message: firstIdx === lastIdx
+                        ? `Parsing chunk ${firstIdx + 1}/${numChunks}...`
+                        : `Parsing chunks ${firstIdx + 1}-${lastIdx + 1}/${numChunks}...`,
+                    detail: filePath,
+                    stats: {
+                        filesProcessed: globalCurrent,
+                        totalFiles: totalParseable,
+                        nodesCreated: graph.nodeCount,
+                    },
+                });
+            };
+            const activeWorkerPool = getOrCreateWorkerPool();
+            if (verboseThroughputLog) {
+                logger.info(`🚚 round: ${misses.length} chunk(s) ${firstIdx + 1}-${lastIdx + 1}/${numChunks}, ` +
+                    `${roundFiles} files in one dispatch`);
+            }
+            const results = dispatchChunkParseRound(misses.map((miss) => ({
+                items: miss.chunkFiles,
+                chunkHash: miss.chunkHash ?? undefined,
+            })), activeWorkerPool, progressForRound);
+            // Mark handled so a rejection during the overlap drain below isn't
+            // flagged as unhandled; the `await` in drainRound re-throws it for real
+            // handling.
+            results.catch(() => { });
+            return { entries, results };
+        };
+        /**
+         * Merge + finalize every chunk of a parked round, in `chunkIdx` order.
+         * Takes RESOLVED worker output: the round's dispatch must already have
+         * settled before this runs, because the pool allows only one dispatch in
+         * flight at a time (see `closeRound`).
+         */
+        const drainRound = async (round) => {
+            const missResults = round.missResults;
+            const missCount = round.entries.reduce((sum, entry) => sum + (entry.kind === 'miss' ? 1 : 0), 0);
+            // `dispatchGroups` returns one array per input group. If that contract
+            // ever breaks, every later entry in this round would silently merge the
+            // wrong chunk's results and skip its cache write, with a clean exit.
+            if (missResults.length !== missCount) {
+                throw new Error(`Parse round result mismatch: ${missResults.length} result group(s) for ${missCount} dispatched chunk(s).`);
+            }
+            let missIdx = 0;
+            for (const entry of round.entries) {
+                if (entry.kind === 'hit') {
+                    const chunkWorkerData = mergeChunkResults(graph, symbolTable, entry.cachedRaw, exportedTypeMap);
+                    await applyChunkResults(chunkWorkerData, entry.chunkIdx, entry.fileCount, entry.chunkStartMs);
+                    continue;
+                }
+                await finalizeWorkerChunk(entry, missResults[missIdx++]);
+            }
+        };
+        /**
+         * Close the accumulated round.
+         *
+         * `WorkerPool.dispatch`/`dispatchGroups` is NOT reentrant — concurrent
+         * calls race on the shared per-slot busy/in-flight state and wedge the
+         * pool until every worker idle-times out. So exactly one dispatch is in
+         * flight here: start this round, merge the PREVIOUS round (whose results
+         * are already resolved) while these workers run, then await this round and
+         * park it resolved for the next close to merge.
+         */
+        const closeRound = async () => {
+            const started = await startRound(roundEntries);
+            roundEntries = [];
+            roundMissBytes = 0;
+            roundBufferedBytes = 0;
+            const previous = pendingRound;
+            pendingRound = null;
+            if (previous) {
+                try {
+                    await drainRound(previous);
+                }
+                catch (err) {
+                    // The round started above is still on the workers. Unwinding now
+                    // reaches this function's `finally`, which calls `terminate()` — and
+                    // terminate kills busy workers outright, which is the #2432
+                    // mid-N-API SIGABRT hazard. Let the in-flight round settle first so
+                    // the pool is idle, then propagate the original failure.
+                    await started?.results.catch(() => undefined);
+                    throw err;
+                }
+            }
+            if (!started)
+                return;
+            let missResults;
+            try {
+                missResults = await started.results;
+            }
+            catch (err) {
+                if (!(err instanceof WorkerPoolInitializationError))
+                    throw err;
+                // Every worker crashed during startup and the pool's bounded self-heal
+                // was exhausted. Fail fast (#1741) — there is no sequential parser to
+                // degrade to. `handleWorkerStartupFailure` always throws, so
+                // `missResults` stays definitely assigned for the parked round below.
+                handleWorkerStartupFailure(err);
+            }
+            pendingRound = { entries: started.entries, missResults };
         };
         for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
             if (heapProbeEveryN > 0 && chunkIdx > 0 && chunkIdx % heapProbeEveryN === 0) {
@@ -812,13 +991,8 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                 // Cache hit: replay cached worker output. Finalize any parked worker
                 // chunk FIRST so deferred aggregation stays in chunk order, then merge
                 // + apply this hit inline (no worker dispatch to overlap).
-                if (pendingWorkerChunk) {
-                    await finalizeWorkerChunk(pendingWorkerChunk);
-                    pendingWorkerChunk = null;
-                }
                 chunkCacheHits++;
                 parseCacheHitFileCount += chunkFiles.length;
-                const chunkWorkerData = mergeChunkResults(graph, symbolTable, cachedRaw, exportedTypeMap);
                 if (isDev) {
                     logger.info(`📦 parse-cache HIT: chunk ${chunkIdx + 1}/${numChunks} (${chunkFiles.length} files, ${chunkHash?.slice(0, 8) ?? 'unknown'})`);
                 }
@@ -830,85 +1004,45 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                     // takes 70-95 so the UI advances through the (potentially long)
                     // resolution stages instead of holding at 82 (M2 from PR #1693
                     // review).
-                    percent: Math.round(20 + ((filesParsedSoFar + cachedFiles) / totalParseable) * 50),
+                    percent: Math.round(20 + ((queuedFilesSoFar + cachedFiles) / totalParseable) * 50),
                     message: `Parsing chunk ${chunkIdx + 1}/${numChunks} (cache)...`,
                     stats: {
-                        filesProcessed: filesParsedSoFar + cachedFiles,
+                        filesProcessed: queuedFilesSoFar + cachedFiles,
                         totalFiles: totalParseable,
                         nodesCreated: graph.nodeCount,
                     },
                 });
                 // The durable gate already snapshotted warm `.v8` shards into the
-                // run-scoped store for scope resolution.
-                await applyChunkResults(chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs);
+                // run-scoped store for scope resolution. Queue into the round so this
+                // hit still finalizes in `chunkIdx` order relative to its neighbours.
+                roundEntries.push({
+                    kind: 'hit',
+                    chunkIdx,
+                    fileCount: chunkFiles.length,
+                    chunkStartMs,
+                    cachedRaw,
+                });
+                for (const file of chunkFiles)
+                    roundBufferedBytes += file.content.length;
+                queuedFilesSoFar += chunkFiles.length;
             }
             else {
-                // Cache miss: dispatch to workers, capture the raw results, store
-                // them under the chunk hash for the next run.
+                // Cache miss: queue for the round's single dispatch; the raw results
+                // are stored under the chunk hash when the round drains.
                 chunkCacheMisses++;
                 reparsedFileCount += chunkFiles.length;
-                if (durableParsedFileDir !== undefined && chunkHash !== null) {
-                    try {
-                        await prepareDurableParsedFileChunk(durableParsedFileDir, chunkHash);
-                    }
-                    catch (err) {
-                        // The durable store is an optimization — degrade like the restore
-                        // path does instead of failing the analyze. Workers recreate the
-                        // directory on write, so at worst the old generation lingers.
-                        logger.warn({ err, chunkHash: chunkHash.slice(0, 8) }, 'parsedfile-cache: could not reset durable chunk generation; continuing');
-                    }
-                }
-                const progressForChunk = (current, _total, filePath) => {
-                    const globalCurrent = filesParsedSoFar + current;
-                    // Parse phase covers 20-70 (M2). Deferred extraction handles 70-95.
-                    const parsingProgress = 20 + (globalCurrent / totalParseable) * 50;
-                    onProgress({
-                        phase: 'parsing',
-                        percent: Math.round(parsingProgress),
-                        message: `Parsing chunk ${chunkIdx + 1}/${numChunks}...`,
-                        detail: filePath,
-                        stats: {
-                            filesProcessed: globalCurrent,
-                            totalFiles: totalParseable,
-                            nodesCreated: graph.nodeCount,
-                        },
-                    });
-                };
-                const activeWorkerPool = getOrCreateWorkerPool();
-                // Worker path — PIPELINE: kick off this chunk's dispatch, merge the
-                // PREVIOUS chunk while these workers parse, then park this chunk for
-                // the next iteration to merge (overlapping its parse). The deferred
-                // merge + parse-cache write-guard + aggregation all run in
-                // `finalizeWorkerChunk`, in chunk order. The pool is the sole parse
-                // path — `getOrCreateWorkerPool` returns a pool or throws.
-                const dispatchPromise = dispatchChunkParse(chunkFiles, activeWorkerPool, progressForChunk, undefined, chunkHash ?? undefined);
-                // Mark handled so a rejection during the overlap drain below isn't
-                // flagged as unhandled; the `await` re-throws it for real handling.
-                dispatchPromise.catch(() => { });
-                if (pendingWorkerChunk) {
-                    await finalizeWorkerChunk(pendingWorkerChunk);
-                    pendingWorkerChunk = null;
+                roundEntries.push({ kind: 'miss', chunkIdx, chunkHash, chunkFiles, chunkStartMs });
+                for (const file of chunkFiles) {
+                    roundMissBytes += file.content.length;
+                    roundBufferedBytes += file.content.length;
                 }
-                let chunkResults;
-                try {
-                    chunkResults = await dispatchPromise;
-                }
-                catch (err) {
-                    if (!(err instanceof WorkerPoolInitializationError))
-                        throw err;
-                    // Every worker crashed during startup and the pool's bounded
-                    // self-heal was exhausted. Fail fast (#1741) — there is no sequential
-                    // parser to degrade to. `handleWorkerStartupFailure` always throws, so
-                    // `chunkResults` stays definitely assigned for the parked chunk below.
-                    handleWorkerStartupFailure(err);
-                }
-                pendingWorkerChunk = {
-                    rawResults: chunkResults,
-                    chunkIdx,
-                    chunkHash,
-                    chunkFiles,
-                    chunkStartMs,
-                };
+                queuedFilesSoFar += chunkFiles.length;
+            }
+            // Close on EITHER cap. `roundMissBytes` sizes the worker round;
+            // `roundBufferedBytes` bounds what the main thread is holding, which is
+            // the only cap a warm run can ever reach.
+            if (roundMissBytes >= roundByteBudget || roundBufferedBytes >= roundByteBudget) {
+                await closeRound();
             }
             // (Per-chunk aggregation + parse-cache write + throughput log now run in
             // `applyChunkResults` / `finalizeWorkerChunk` — see the merge-pipelining
@@ -916,11 +1050,14 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
             // resolution in the single end-of-loop pass below, the rest by the
             // scope-resolution phase, RING4-2 #943.)
         }
-        // Drain the final parked worker chunk — the last pipelined chunk has no
-        // successor to overlap its merge with, so merge + finalize it here.
-        if (pendingWorkerChunk) {
-            await finalizeWorkerChunk(pendingWorkerChunk);
-            pendingWorkerChunk = null;
+        // Drain the tail: close the partially-filled round, then drain the round
+        // it parked — the last round has no successor to overlap its merge with.
+        if (roundEntries.length > 0)
+            await closeRound();
+        if (pendingRound) {
+            const last = pendingRound;
+            pendingRound = null;
+            await drainRound(last);
         }
         if (isDev && parseCache && (chunkCacheHits > 0 || chunkCacheMisses > 0)) {
             logger.info(`📦 parse-cache summary: ${chunkCacheHits} chunk hit(s), ${chunkCacheMisses} miss(es) across ${numChunks} chunk(s)`);

package/dist/core/ingestion/workers/parse-worker.js

@@ -993,11 +993,22 @@ function reportWarning(message) {
         logger.warn(message);
     }
 }
+// Keep compiled queries across jobs in this worker. A language can select
+// multiple native grammars, so both grammar identity and query text matter.
+const compiledQueries = new WeakMap();
 const processFileGroup = (files, language, queryString, result, onFileProcessed) => {
     let query;
     try {
         const lang = parser.getLanguage();
-        query = new Parser.Query(lang, queryString);
+        let queries = compiledQueries.get(lang);
+        if (!queries) {
+            queries = new Map();
+            compiledQueries.set(lang, queries);
+        }
+        const cached = queries.get(queryString);
+        query = cached ?? new Parser.Query(lang, queryString);
+        if (!cached)
+            queries.set(queryString, query);
     }
     catch (err) {
         reportWarning(`Query compilation failed for ${language}: ${err instanceof Error ? err.message : String(err)}`);

package/dist/core/ingestion/workers/worker-pool.d.ts

@@ -1,8 +1,38 @@
 import { Worker } from 'node:worker_threads';
+/**
+ * One content-addressed parse-cache chunk's worth of work inside a pool round.
+ * See {@link WorkerPool.dispatchGroups}.
+ */
+export interface DispatchGroup<TInput> {
+    readonly items: readonly TInput[];
+    /**
+     * Chunk hash tagged onto every job derived from `items`, exactly as the
+     * `chunkHash` argument of {@link WorkerPool.dispatch} does for a lone chunk.
+     */
+    readonly chunkHash?: string;
+}
 export interface WorkerPool {
     /**
-     * Dispatch items across workers. Items are split into bounded jobs, each job
-     * is committed independently, and stalled jobs are split/retried locally.
+     * Dispatch several content-addressed chunks in ONE pool round.
+     *
+     * `dispatch` is a barrier: it resolves only once every job it created has
+     * committed, so dispatching one small parse-cache pack at a time leaves most
+     * slots idle for the whole round-trip. Stable packs are keyed by
+     * `(language, hash(path) % 128)`, which routinely yields packs far below the
+     * byte budget — on this repo, 1285 packs where the budget alone needs 16, and
+     * 549 of them hold a single file. Batching packs into one round removes those
+     * barriers without touching pack identity: jobs are still cut at group
+     * boundaries, so each job carries exactly one `chunkHash` and every result
+     * stays attributable to the pack that owns its cache key.
+     *
+     * Returns one result array per input group, in input order. A group whose
+     * items were all quarantined yields an empty array.
+     */
+    dispatchGroups<TInput, TResult>(groups: readonly DispatchGroup<TInput>[], onProgress?: (filesProcessed: number) => void): Promise<TResult[][]>;
+    /**
+     * Dispatch ONE chunk across workers — {@link WorkerPool.dispatchGroups} with
+     * a single group. Items are split into bounded jobs, each job is committed
+     * independently, and stalled jobs are split/retried locally.
      *
      * Files in {@link WorkerPool.getQuarantinedPaths} are filtered out before
      * dispatch — they have already caused a worker death this pool lifetime and

package/dist/core/ingestion/workers/worker-pool.js

@@ -488,9 +488,14 @@ function inFlightExcludePath(job, lastProgress) {
     const path = itemPath(job.items[lastProgress]);
     return path ? [path] : [];
 }
-function createJobs(items, maxItems, maxBytes, timeoutMs, chunkHash) {
+/**
+ * Cut `items` into bounded jobs. `startIndexOffset` places those jobs on a
+ * shared index space so several groups can be laid out end to end in one
+ * dispatch round and every result still sorts back into global input order.
+ */
+function createJobs(items, maxItems, maxBytes, timeoutMs, chunkHash, startIndexOffset = 0) {
     const jobs = [];
-    let startIndex = 0;
+    let startIndex = startIndexOffset;
     let batch = [];
     let batchBytes = 0;
     const flush = () => {
@@ -827,7 +832,31 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
     // First dispatch awaits this; it settles every slot's bounded retry loop in
     // parallel and drops the unrecoverable ones before any dispatch can fire.
     const initialReadyGate = Promise.allSettled(workers.map((_, i) => bringSlotReady(i))).then(() => undefined);
-    const dispatch = async (items, onProgress, chunkHash) => {
+    /**
+     * Guards the one-dispatch-at-a-time contract. The dispatch machinery keeps
+     * its jobs/busy-slot/in-flight state per call, so two concurrent dispatches
+     * hand the same slots out twice: both stall, and the failure surfaces only
+     * when every worker hits its idle timeout (10s+ of a wedged pool with no
+     * indication of the cause). Fail loudly at the call instead.
+     */
+    let dispatchInFlight = false;
+    /**
+     * Claim the pool synchronously, then run the dispatch. The claim CANNOT be
+     * taken inside `dispatchGroupsInner`: its first statement awaits the
+     * readiness gate, so two calls made in the same tick would both get past the
+     * check before either set the flag.
+     */
+    const dispatchGroups = (groups, onProgress) => {
+        if (dispatchInFlight) {
+            return Promise.reject(new WorkerPoolDispatchError('Worker pool dispatch is already in flight. `dispatch`/`dispatchGroups` is not ' +
+                'reentrant — await the previous call before starting another on the same pool.', []));
+        }
+        dispatchInFlight = true;
+        return dispatchGroupsInner(groups, onProgress).finally(() => {
+            dispatchInFlight = false;
+        });
+    };
+    const dispatchGroupsInner = async (groups, onProgress) => {
         // Await the initial-spawn readiness gate (F13). On first dispatch
         // this blocks for up to poolOptions.workerReadyTimeoutMs while every initial
         // worker's `{type:'ready'}` handshake is checked; on subsequent
@@ -842,8 +871,9 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
             throw new WorkerPoolDispatchError(`Worker pool circuit breaker tripped${reason}. ` +
                 `Subsequent dispatches require a fresh pool instance.`, []);
         }
-        if (items.length === 0)
-            return [];
+        const emptyPerGroup = () => groups.map(() => []);
+        if (groups.every((group) => group.items.length === 0))
+            return emptyPerGroup();
         if (activeSlots.size === 0) {
             const detail = initialReadinessFailures.length > 0
                 ? ` after initial ready handshake: ${initialReadinessFailures.join('; ')}`
@@ -859,17 +889,38 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
         // Layer 3: filter out quarantined paths so a known-bad file never reaches
         // a worker again this pool lifetime. The caller queries
         // `getQuarantinedPaths` after dispatch to route filtered items.
-        const dispatchableItems = [];
-        for (const item of items) {
-            const path = itemPath(item);
-            if (path !== undefined && quarantine.has(path))
-                continue;
-            dispatchableItems.push(item);
+        const dispatchableGroups = groups.map((group) => {
+            const items = [];
+            for (const item of group.items) {
+                const path = itemPath(item);
+                if (path !== undefined && quarantine.has(path))
+                    continue;
+                items.push(item);
+            }
+            return { items, chunkHash: group.chunkHash };
+        });
+        const dispatchableCount = dispatchableGroups.reduce((sum, group) => sum + group.items.length, 0);
+        if (dispatchableCount === 0)
+            return emptyPerGroup();
+        // Stable cache packs can be much smaller than either job ceiling. Split
+        // those packs across the live slots too, otherwise each serial dispatch
+        // feeds only one worker. Keep both configured ceilings as upper bounds.
+        const maxItemsPerJob = Math.min(poolOptions.subBatchSize, Math.max(1, Math.floor(dispatchableCount / activeSlots.size)));
+        // Lay the groups end to end on one index space and cut jobs at every group
+        // boundary. A job therefore belongs to exactly one group, which is what
+        // lets a result be attributed back to the parse-cache chunk that owns it
+        // (and what keeps `chunkHash` a per-job constant through splits/requeues).
+        const jobs = [];
+        const groupEnds = [];
+        let groupStart = 0;
+        for (const group of dispatchableGroups) {
+            for (const job of createJobs(group.items, maxItemsPerJob, poolOptions.subBatchMaxBytes, poolOptions.subBatchIdleTimeoutMs, group.chunkHash, groupStart)) {
+                jobs.push(job);
+            }
+            groupStart += group.items.length;
+            groupEnds.push(groupStart);
         }
-        if (dispatchableItems.length === 0)
-            return [];
-        const jobs = createJobs(dispatchableItems, poolOptions.subBatchSize, poolOptions.subBatchMaxBytes, poolOptions.subBatchIdleTimeoutMs, chunkHash);
-        return new Promise((resolve, reject) => {
+        return await new Promise((resolve, reject) => {
             const results = [];
             const inFlightProgress = new Array(size).fill(0);
             // Tracks which slots are currently mid-job so the "wake idle slots"
@@ -899,7 +950,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                 if (!onProgress)
                     return;
                 const inFlight = inFlightProgress.reduce((sum, value) => sum + value, 0);
-                const next = Math.min(dispatchableItems.length, Math.max(maxReported, completedFiles + inFlight));
+                const next = Math.min(dispatchableCount, Math.max(maxReported, completedFiles + inFlight));
                 if (next === maxReported)
                     return;
                 maxReported = next;
@@ -984,7 +1035,13 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                     retireWorkerAfterTimeout(existing, workerIndex, reason);
                     return;
                 }
-                await existing.terminate().catch(() => undefined);
+                // Recovery must settle before dispatch returns, but a failed thread
+                // may never acknowledge termination. Bound that wait as in shutdown.
+                const termination = existing.terminate().then(() => undefined, () => undefined);
+                if (!(await settledWithin(termination, poolOptions.shutdownDrainMs))) {
+                    existing.unref?.();
+                    logger.warn({ workerIndex, drainMs: poolOptions.shutdownDrainMs, reason }, `Worker ${workerIndex} did not finish terminating within the shutdown drain; continuing recovery.`);
+                }
             };
             const replaceWorker = async (workerIndex, mode = 'terminate', reason = 'replacing worker') => {
                 await removeWorkerFromSlot(workerIndex, mode, reason);
@@ -1057,9 +1114,20 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                 if (jobs.length === 0 && activeWorkers === 0) {
                     stopped = true;
                     results.sort((a, b) => a.startIndex - b.startIndex);
-                    if (onProgress && maxReported < dispatchableItems.length)
-                        onProgress(dispatchableItems.length);
-                    resolve(results.map((result) => result.data));
+                    if (onProgress && maxReported < dispatchableCount)
+                        onProgress(dispatchableCount);
+                    // Partition back per group. Job (and split sub-job) start indices
+                    // stay inside their group's span, so a single forward walk over the
+                    // sorted results assigns every result to exactly one group.
+                    const perGroup = groupEnds.map(() => []);
+                    let groupIdx = 0;
+                    for (const result of results) {
+                        while (groupIdx < groupEnds.length - 1 && result.startIndex >= groupEnds[groupIdx]) {
+                            groupIdx++;
+                        }
+                        perGroup[groupIdx].push(result.data);
+                    }
+                    resolve(perGroup);
                 }
             };
             // Re-queue the non-quarantined remainder of a dead worker's job so a
@@ -1357,11 +1425,13 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                 // (`error`, `exit`, msg-channel error). Bridges the per-job teardown
                 // into the pool-level handleWorkerDeath recovery + breaker logic.
                 const recoverAndResume = async (reason, excludePaths) => {
-                    activeWorkers--;
                     busySlots.delete(workerIndex);
                     inFlightProgress[workerIndex] = 0;
                     requeueRemainder(job, excludePaths);
+                    // Keep recovery in flight so another slot finishing cannot settle
+                    // this dispatch before the replacement is ready for the next one.
                     await handleWorkerDeath(workerIndex, reason, excludePaths);
+                    activeWorkers--;
                     if (stopped)
                         return;
                     // Slot may have been dropped or respawned. Kick the current slot
@@ -1406,10 +1476,10 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                                 // is respawned (or dropped) and can dispatch the next
                                 // job deterministically.
                                 void (async () => {
-                                    activeWorkers--;
                                     busySlots.delete(workerIndex);
                                     requeueRemainder(job, decision.excludePaths);
                                     await handleWorkerDeath(workerIndex, decision.reason, decision.excludePaths, 'retire');
+                                    activeWorkers--;
                                     if (stopped)
                                         return;
                                     if (activeSlots.has(workerIndex))
@@ -1689,8 +1759,13 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
         workers.length = 0;
         activeSlots.clear();
     };
+    const dispatch = async (items, onProgress, chunkHash) => {
+        const [result] = await dispatchGroups([{ items, chunkHash }], onProgress);
+        return result ?? [];
+    };
     return {
         dispatch,
+        dispatchGroups,
         terminate,
         size,
         getQuarantinedPaths: () => quarantine.snapshot(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.12-rc.3",
+  "version": "1.6.12-rc.4",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",