gitnexus 1.6.11 → 1.6.12-rc.10

package/dist/core/ingestion/pipeline-phases/parse-impl.js

@@ -15,7 +15,7 @@
  * @module
  */
 import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
-import { mergeChunkResults, dispatchChunkParse } from '../parsing-processor.js';
+import { mergeChunkResults, dispatchChunkParseRound } from '../parsing-processor.js';
 import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
 import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, } from '../../../storage/parsedfile-store.js';
 import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
@@ -28,7 +28,7 @@ import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
 import { getProvider, getProviderForFile, providers } from '../languages/index.js';
 import { SCOPE_RESOLVERS } from '../scope-resolution/pipeline/registry.js';
 import { DATA_ROUTE_TABLE_SOURCE } from '../route-extractors/data-route-table.js';
-import { createWorkerPool, workerPoolDisabledByEnv, resolveAutoPoolSize, WorkerPoolInitializationError, WorkerPoolDisabledError, } from '../workers/worker-pool.js';
+import { createWorkerPool, workerPoolDisabledByEnv, resolveAutoPoolSize, envWorkerPoolSize, resolveHostParallelism, WorkerPoolInitializationError, WorkerPoolDisabledError, } from '../workers/worker-pool.js';
 import { normalizeExtractedRoutePath } from '../route-extractors/route-path.js';
 import { resolveOperands } from '../route-extractors/python-const-resolver.js';
 import { prepareRouteConstantsByProvider } from '../language-provider.js';
@@ -43,6 +43,8 @@ import { isVerboseIngestionEnabled } from '../utils/verbose.js';
 import { endTimer, isDeferredResolutionProfileEnabled, logDeferredProfile, startTimer, } from '../utils/deferred-resolution-profile.js';
 import { isDebugHeapEnabled, logHeapProbe } from '../utils/heap-probe.js';
 import { logger } from '../../logger.js';
+import { mapConcurrent } from '../../../lib/utils.js';
+import { createRoundBudget } from './parse-round-budget.js';
 // ── Constants ──────────────────────────────────────────────────────────────
 /**
  * Heap-scale guardrail constants (#2649). Measured on a Linux-kernel analyze:
@@ -130,8 +132,36 @@ const CHUNK_BYTES_PER_WORKER = DEFAULT_CHUNK_BYTE_BUDGET;
  * while the rest finish early). Drives the derived `subBatchMaxBytes`.
  */
 const TARGET_JOBS_PER_WORKER = 3;
+/**
+ * Concurrent durable ParsedFile directory resets per round. Matches the file
+ * reader's `READ_CONCURRENCY`, because both compete for the same descriptors.
+ */
+const DURABLE_RESET_CONCURRENCY = 32;
 /** Floor for a derived sub-batch so jobs don't shrink to per-file IPC churn. */
 const MIN_SUB_BATCH_BYTES = 256 * 1024;
+/**
+ * Source bytes an open round may HOLD — cache hits and misses alike — before
+ * it is dispatched and drained.
+ *
+ * A `dispatch` is a barrier, so one round-trip per cache pack leaves most slots
+ * idle: packs are keyed by `(language, hash(path) % 128)` and routinely land far
+ * under {@link DEFAULT_CHUNK_BYTE_BUDGET} (this repo: 1285 packs where the byte
+ * budget alone needs 16, 549 of them holding a single file). Rounds batch packs
+ * into one `dispatchGroups` call without touching pack identity.
+ *
+ * This is the in-flight cap, the same role Piscina's `maxQueue` plays: bigger
+ * rounds remove more barriers but hold more file content and more un-merged
+ * worker output on the main thread at once. Defaulting to one chunk budget
+ * keeps in-flight source bytes at the magnitude the loop already prefetched
+ * (`parseChunkConcurrency`, 2 chunks ahead). Override via
+ * `GITNEXUS_PARSE_ROUND_BYTES`.
+ */
+function resolveParseRoundByteBudget(options) {
+    const env = Number(process.env.GITNEXUS_PARSE_ROUND_BYTES);
+    if (Number.isFinite(env) && env > 0)
+        return env;
+    return resolveChunkByteBudget(options);
+}
 function resolveChunkByteBudget(options) {
     const opt = options?.chunkByteBudget;
     if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0)
@@ -391,11 +421,38 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
     // cores-based auto size is capped by source bytes / CHUNK_BYTES_PER_WORKER
     // so a tiny repo does not spawn a full idle pool. Cache pack membership
     // is independent of this number (#3088).
-    const explicitPoolSize = options?.workerPoolSize;
+    // `--workers <N>` and `GITNEXUS_WORKER_POOL_SIZE` are both deliberate
+    // operator input, so both bypass the work-proportional cap below. Only the
+    // env path used to be clamped by it, which made the documented escape hatch
+    // silently do nothing: on a 30MB repo the cap resolves to 16, so an operator
+    // asking for 24 still got 16 with no warning, while `--workers 24` got 24.
+    const explicitPoolSize = options?.workerPoolSize ?? envWorkerPoolSize();
+    // Cores-based auto size, bounded by source bytes so a tiny repo does not
+    // spawn a full idle pool.
     const workProportionalCap = Math.max(1, Math.ceil(totalBytes / CHUNK_BYTES_PER_WORKER));
+    // An operator's number is honored, but never exceeds the number of files
+    // there are to parse — `GITNEXUS_WORKER_POOL_SIZE=100000` on a five-file repo
+    // should not become the literal thread count. This bounds `--workers` and the
+    // env var identically, keeping the parity above intact. Note it does NOT
+    // shrink an incremental re-analyze: `totalParseable` counts every parseable
+    // file in the scan, not the changed ones, so a warm run of a large repo still
+    // spawns the full requested pool.
     const effectivePoolSize = explicitPoolSize && explicitPoolSize > 0
-        ? explicitPoolSize
+        ? Math.min(explicitPoolSize, Math.max(1, totalParseable))
         : Math.min(resolveAutoPoolSize(), workProportionalCap);
+    // Deliberate over-subscription is the operator's call, so this warns rather
+    // than caps — silently capping is what the override exists to stop. But an
+    // exported `GITNEXUS_WORKER_POOL_SIZE` applies to EVERY analyze in a
+    // long-lived caller (watch auto-sync, the MCP server), including small
+    // incremental ones, and that is easy to set once and forget.
+    if (explicitPoolSize && explicitPoolSize > 0) {
+        const hostParallelism = resolveHostParallelism();
+        if (effectivePoolSize > hostParallelism) {
+            logger.warn({ requested: explicitPoolSize, spawning: effectivePoolSize, hostParallelism }, `Worker pool size ${effectivePoolSize} exceeds this host's ${hostParallelism} usable core(s); ` +
+                `parsing is CPU-bound, so the extra workers add memory pressure without throughput. ` +
+                `This applies to every analyze while the override is set.`);
+        }
+    }
     // Cache packs: stable (language, hash(path) mod 128) buckets, then the
     // per-call byte budget inside each bucket (#3088). Pool size is used only
     // for worker count and sub-batch fan-out, not membership.
@@ -596,13 +653,43 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         // the env can't change mid-run.
         const verboseThroughputLog = isDev || isVerboseIngestionEnabled();
         const heapProbeEveryN = isDebugHeapEnabled() ? 25 : 0;
-        let pendingWorkerChunk = null;
+        /**
+         * Chunk hashes whose durable ParsedFile directory could not be reset. The
+         * old generation's shards are still on disk, so a warm hit would union
+         * stale shards with the new ones. Treated exactly like a quarantined chunk:
+         * skip the parse-cache write so the next run re-dispatches into a clean
+         * directory rather than trusting a generation we could not clear.
+         */
+        const durablePrepareFailures = new Set();
+        const roundByteBudget = resolveParseRoundByteBudget(options);
+        let roundEntries = [];
+        /**
+         * Bytes an open round is HOLDING, counting hits as well as misses.
+         *
+         * Counting only the cache-MISSING bytes would bound just what the workers
+         * are asked to do, so a warm run — where nothing misses — would never reach
+         * the close condition and would buffer every chunk's cached output until
+         * the tail drain. That is the #2649 heap failure on a large repo. Counting
+         * both keeps a hits-only run draining at the same cadence as a cold one;
+         * `startRound` already supports a round with no misses.
+         *
+         * Measured in UTF-8 bytes, matching `estimateItemBytes` in the worker pool,
+         * so the cap means the same thing here as it does for a job's payload.
+         */
+        const roundBudget = createRoundBudget(roundByteBudget);
+        /**
+         * Files QUEUED into rounds so far. `filesParsedSoFar` only advances when a
+         * round drains, so it is the right number for the throughput log but would
+         * pin a warm run's progress bar at the phase floor for the whole loop.
+         */
+        let queuedFilesSoFar = 0;
+        let pendingRound = null;
         // Apply one chunk's merged worker data: per-chunk aggregation into the
         // run-level accumulators + the throughput log. Shared by the cache-hit
         // (inline) and worker (deferred) paths. The `| null` guard is defensive —
         // every live caller passes real worker data now that sequential parsing
         // (which was the only path that passed null) is gone.
-        const applyChunkResults = async (chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs) => {
+        const applyChunkResults = async (chunkWorkerData, chunkIdx, fileCount, chunkStartMs) => {
             if (chunkWorkerData) {
                 for (const filePath of chunkWorkerData.scopeExtractionFailures) {
                     scopeExtractionFailures.add(filePath);
@@ -690,30 +777,35 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                         allORMQueries.push(item);
                 }
             }
-            filesParsedSoFar += chunkFiles.length;
+            filesParsedSoFar += fileCount;
             if (verboseThroughputLog && chunkStartMs !== null) {
                 const elapsedMs = Date.now() - chunkStartMs;
-                const filesPerSec = elapsedMs > 0 ? (chunkFiles.length * 1000) / elapsedMs : 0;
+                const filesPerSec = elapsedMs > 0 ? (fileCount * 1000) / elapsedMs : 0;
                 const stats = workerPool?.getStats?.();
                 const poolFrag = stats
                     ? ` pool: ${stats.activeSlots}/${stats.size} active, ` +
                         `${stats.quarantined} quarantined${stats.poolBroken ? ', BROKEN' : ''}`
                     : ' (cache replay)';
-                logger.info(`📊 chunk ${chunkIdx + 1}/${numChunks}: ${chunkFiles.length} files in ${elapsedMs}ms ` +
+                logger.info(`📊 chunk ${chunkIdx + 1}/${numChunks}: ${fileCount} files in ${elapsedMs}ms ` +
                     `(${filesPerSec.toFixed(1)} files/s)${poolFrag}`);
             }
         };
         // Merge + finalize a parked worker chunk: graph merge (the overlapped
         // main-thread step) → parse-cache write-guard → run-level aggregation.
-        const finalizeWorkerChunk = async (p) => {
-            const chunkWorkerData = mergeChunkResults(graph, symbolTable, p.rawResults, exportedTypeMap);
+        const finalizeWorkerChunk = async (p, rawResults) => {
+            const chunkWorkerData = mergeChunkResults(graph, symbolTable, rawResults, exportedTypeMap);
             // Persist raw results for this chunk hash (skipping when any chunk file
             // was worker-quarantined, so the narrower rawResults isn't cached under
             // the full-chunk key — see the original inline note / U20.U2).
-            if (parseCache && p.chunkHash && p.rawResults.length > 0) {
+            if (parseCache && p.chunkHash && rawResults.length > 0) {
                 const quarantineSet = new Set(workerPool?.getQuarantinedPaths?.() ?? []);
                 const chunkHadQuarantine = p.chunkFiles.some((f) => quarantineSet.has(f.path));
-                if (chunkHadQuarantine) {
+                const durableGenerationStale = durablePrepareFailures.has(p.chunkHash);
+                if (durableGenerationStale) {
+                    logger.warn({ chunkHash: p.chunkHash.slice(0, 8) }, 'parse-cache SKIP: durable generation for this chunk could not be reset, ' +
+                        'so its shards may be stale; next run will re-dispatch it');
+                }
+                else if (chunkHadQuarantine) {
                     if (isDev) {
                         const quarantinedInChunk = p.chunkFiles.filter((f) => quarantineSet.has(f.path)).length;
                         logger.info(`📦 parse-cache SKIP: chunk ${p.chunkIdx + 1}/${numChunks} ` +
@@ -722,13 +814,164 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                     }
                 }
                 else {
-                    await persistParseCacheChunk(parseCache, p.chunkHash, p.rawResults);
+                    await persistParseCacheChunk(parseCache, p.chunkHash, rawResults);
                     if (isDev) {
                         logger.info(`📦 parse-cache MISS+store: chunk ${p.chunkIdx + 1}/${numChunks} (${p.chunkFiles.length} files, ${p.chunkHash.slice(0, 8)})`);
                     }
                 }
             }
-            await applyChunkResults(chunkWorkerData, p.chunkIdx, p.chunkFiles, p.chunkStartMs);
+            await applyChunkResults(chunkWorkerData, p.chunkIdx, p.chunkFiles.length, p.chunkStartMs);
+        };
+        /**
+         * Dispatch a round's cache misses as ONE pool round. Returns the parked
+         * round; the caller drains it after starting the next one so the workers
+         * parse round N+1 while the main thread merges round N (the same overlap
+         * the per-chunk loop had, at round granularity).
+         */
+        const startRound = async (entries) => {
+            if (entries.length === 0)
+                return null;
+            const misses = entries.filter((entry) => entry.kind === 'miss');
+            if (misses.length === 0) {
+                return { entries, results: Promise.resolve([]) };
+            }
+            // Each chunk resets its own directory, so these are independent and run
+            // concurrently: serially they would sit on the critical path this round
+            // exists to shorten, with the pool idle and the previous round's merge
+            // waiting, once per miss.
+            //
+            // BOUNDED, though. A round can hold hundreds of small packs, and each
+            // reset is a recursive rm + mkdir. Firing all of them at once competes
+            // for descriptors with the chunk prefetch this loop already has in
+            // flight, and `readFileContents` degrades a losing read SILENTLY by
+            // contract — a dropped file would vanish from the chunk, from the graph,
+            // and from the chunk hash, shipping a narrowed index with exit 0. Same
+            // helper and width the file reads use.
+            await mapConcurrent(misses, async (miss) => {
+                if (durableParsedFileDir === undefined || miss.chunkHash === null)
+                    return;
+                try {
+                    await prepareDurableParsedFileChunk(durableParsedFileDir, miss.chunkHash);
+                }
+                catch (err) {
+                    // The durable store is an optimization — degrade like the restore
+                    // path does instead of failing the analyze. Workers recreate the
+                    // directory on write, so at worst the old generation lingers.
+                    // Caught per chunk so one failure cannot abort the others.
+                    durablePrepareFailures.add(miss.chunkHash);
+                    logger.warn({ err, chunkHash: miss.chunkHash.slice(0, 8) }, 'parsedfile-cache: could not reset durable chunk generation; ' +
+                        'continuing without caching this chunk');
+                }
+            }, { concurrency: DURABLE_RESET_CONCURRENCY });
+            const roundFiles = misses.reduce((sum, miss) => sum + miss.chunkFiles.length, 0);
+            const firstIdx = misses[0].chunkIdx;
+            const lastIdx = misses[misses.length - 1].chunkIdx;
+            const progressForRound = (current, _total, filePath) => {
+                // Rounds queued before this one are already counted in
+                // `queuedFilesSoFar`; `current` is this round's own worker progress.
+                const globalCurrent = queuedFilesSoFar - roundFiles + current;
+                // Parse phase covers 20-70 (M2). Deferred extraction handles 70-95.
+                const parsingProgress = 20 + (globalCurrent / totalParseable) * 50;
+                onProgress({
+                    phase: 'parsing',
+                    percent: Math.round(parsingProgress),
+                    message: firstIdx === lastIdx
+                        ? `Parsing chunk ${firstIdx + 1}/${numChunks}...`
+                        : `Parsing chunks ${firstIdx + 1}-${lastIdx + 1}/${numChunks}...`,
+                    detail: filePath,
+                    stats: {
+                        filesProcessed: globalCurrent,
+                        totalFiles: totalParseable,
+                        nodesCreated: graph.nodeCount,
+                    },
+                });
+            };
+            const activeWorkerPool = getOrCreateWorkerPool();
+            if (verboseThroughputLog) {
+                logger.info(`🚚 round: ${misses.length} chunk(s) ${firstIdx + 1}-${lastIdx + 1}/${numChunks}, ` +
+                    `${roundFiles} files in one dispatch`);
+            }
+            const results = dispatchChunkParseRound(misses.map((miss) => ({
+                items: miss.chunkFiles,
+                chunkHash: miss.chunkHash ?? undefined,
+            })), activeWorkerPool, progressForRound);
+            // Mark handled so a rejection during the overlap drain below isn't
+            // flagged as unhandled; the `await` in drainRound re-throws it for real
+            // handling.
+            results.catch(() => { });
+            return { entries, results };
+        };
+        /**
+         * Merge + finalize every chunk of a parked round, in `chunkIdx` order.
+         * Takes RESOLVED worker output: the round's dispatch must already have
+         * settled before this runs, because the pool allows only one dispatch in
+         * flight at a time (see `closeRound`).
+         */
+        const drainRound = async (round) => {
+            const missResults = round.missResults;
+            const missCount = round.entries.filter((entry) => entry.kind === 'miss').length;
+            // `dispatchGroups` returns one array per input group. If that contract
+            // ever breaks, every later entry in this round would silently merge the
+            // wrong chunk's results and skip its cache write, with a clean exit.
+            if (missResults.length !== missCount) {
+                throw new Error(`Parse round result mismatch: ${missResults.length} result group(s) for ${missCount} dispatched chunk(s).`);
+            }
+            let missIdx = 0;
+            for (const entry of round.entries) {
+                if (entry.kind === 'hit') {
+                    const chunkWorkerData = mergeChunkResults(graph, symbolTable, entry.cachedRaw, exportedTypeMap);
+                    await applyChunkResults(chunkWorkerData, entry.chunkIdx, entry.fileCount, entry.chunkStartMs);
+                    continue;
+                }
+                await finalizeWorkerChunk(entry, missResults[missIdx++]);
+            }
+        };
+        /**
+         * Close the accumulated round.
+         *
+         * `WorkerPool.dispatch`/`dispatchGroups` is NOT reentrant — concurrent
+         * calls race on the shared per-slot busy/in-flight state and wedge the
+         * pool until every worker idle-times out. So exactly one dispatch is in
+         * flight here: start this round, merge the PREVIOUS round (whose results
+         * are already resolved) while these workers run, then await this round and
+         * park it resolved for the next close to merge.
+         */
+        const closeRound = async () => {
+            const started = await startRound(roundEntries);
+            roundEntries = [];
+            roundBudget.reset();
+            const previous = pendingRound;
+            pendingRound = null;
+            if (previous) {
+                try {
+                    await drainRound(previous);
+                }
+                catch (err) {
+                    // The round started above is still on the workers. Unwinding now
+                    // reaches this function's `finally`, which calls `terminate()` — and
+                    // terminate kills busy workers outright, which is the #2432
+                    // mid-N-API SIGABRT hazard. Let the in-flight round settle first so
+                    // the pool is idle, then propagate the original failure.
+                    await started?.results.catch(() => undefined);
+                    throw err;
+                }
+            }
+            if (!started)
+                return;
+            let missResults;
+            try {
+                missResults = await started.results;
+            }
+            catch (err) {
+                if (!(err instanceof WorkerPoolInitializationError))
+                    throw err;
+                // Every worker crashed during startup and the pool's bounded self-heal
+                // was exhausted. Fail fast (#1741) — there is no sequential parser to
+                // degrade to. `handleWorkerStartupFailure` always throws, so
+                // `missResults` stays definitely assigned for the parked round below.
+                handleWorkerStartupFailure(err);
+            }
+            pendingRound = { entries: started.entries, missResults };
         };
         for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
             if (heapProbeEveryN > 0 && chunkIdx > 0 && chunkIdx % heapProbeEveryN === 0) {
@@ -808,17 +1051,14 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                 parsedFileStorePath !== undefined &&
                 durableExpectedPaths !== undefined &&
                 (await durableChunkHasShards(parsedFileStorePath, chunkHash, durableExpectedPaths));
+            // Set by whichever branch queues this chunk; drives the close below.
+            let roundIsFull = false;
             if (cachedRaw && cachedRaw.length > 0 && (durableHit || parsedFileStorePath === undefined)) {
                 // Cache hit: replay cached worker output. Finalize any parked worker
                 // chunk FIRST so deferred aggregation stays in chunk order, then merge
                 // + apply this hit inline (no worker dispatch to overlap).
-                if (pendingWorkerChunk) {
-                    await finalizeWorkerChunk(pendingWorkerChunk);
-                    pendingWorkerChunk = null;
-                }
                 chunkCacheHits++;
                 parseCacheHitFileCount += chunkFiles.length;
-                const chunkWorkerData = mergeChunkResults(graph, symbolTable, cachedRaw, exportedTypeMap);
                 if (isDev) {
                     logger.info(`📦 parse-cache HIT: chunk ${chunkIdx + 1}/${numChunks} (${chunkFiles.length} files, ${chunkHash?.slice(0, 8) ?? 'unknown'})`);
                 }
@@ -830,97 +1070,55 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                     // takes 70-95 so the UI advances through the (potentially long)
                     // resolution stages instead of holding at 82 (M2 from PR #1693
                     // review).
-                    percent: Math.round(20 + ((filesParsedSoFar + cachedFiles) / totalParseable) * 50),
+                    percent: Math.round(20 + ((queuedFilesSoFar + cachedFiles) / totalParseable) * 50),
                     message: `Parsing chunk ${chunkIdx + 1}/${numChunks} (cache)...`,
                     stats: {
-                        filesProcessed: filesParsedSoFar + cachedFiles,
+                        filesProcessed: queuedFilesSoFar + cachedFiles,
                         totalFiles: totalParseable,
                         nodesCreated: graph.nodeCount,
                     },
                 });
                 // The durable gate already snapshotted warm `.v8` shards into the
-                // run-scoped store for scope resolution.
-                await applyChunkResults(chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs);
+                // run-scoped store for scope resolution. Queue into the round so this
+                // hit still finalizes in `chunkIdx` order relative to its neighbours.
+                roundEntries.push({
+                    kind: 'hit',
+                    chunkIdx,
+                    fileCount: chunkFiles.length,
+                    chunkStartMs,
+                    cachedRaw,
+                });
+                roundIsFull = roundBudget.addChunk(chunkFiles.map((file) => file.content));
+                queuedFilesSoFar += chunkFiles.length;
             }
             else {
-                // Cache miss: dispatch to workers, capture the raw results, store
-                // them under the chunk hash for the next run.
+                // Cache miss: queue for the round's single dispatch; the raw results
+                // are stored under the chunk hash when the round drains.
                 chunkCacheMisses++;
                 reparsedFileCount += chunkFiles.length;
-                if (durableParsedFileDir !== undefined && chunkHash !== null) {
-                    try {
-                        await prepareDurableParsedFileChunk(durableParsedFileDir, chunkHash);
-                    }
-                    catch (err) {
-                        // The durable store is an optimization — degrade like the restore
-                        // path does instead of failing the analyze. Workers recreate the
-                        // directory on write, so at worst the old generation lingers.
-                        logger.warn({ err, chunkHash: chunkHash.slice(0, 8) }, 'parsedfile-cache: could not reset durable chunk generation; continuing');
-                    }
-                }
-                const progressForChunk = (current, _total, filePath) => {
-                    const globalCurrent = filesParsedSoFar + current;
-                    // Parse phase covers 20-70 (M2). Deferred extraction handles 70-95.
-                    const parsingProgress = 20 + (globalCurrent / totalParseable) * 50;
-                    onProgress({
-                        phase: 'parsing',
-                        percent: Math.round(parsingProgress),
-                        message: `Parsing chunk ${chunkIdx + 1}/${numChunks}...`,
-                        detail: filePath,
-                        stats: {
-                            filesProcessed: globalCurrent,
-                            totalFiles: totalParseable,
-                            nodesCreated: graph.nodeCount,
-                        },
-                    });
-                };
-                const activeWorkerPool = getOrCreateWorkerPool();
-                // Worker path — PIPELINE: kick off this chunk's dispatch, merge the
-                // PREVIOUS chunk while these workers parse, then park this chunk for
-                // the next iteration to merge (overlapping its parse). The deferred
-                // merge + parse-cache write-guard + aggregation all run in
-                // `finalizeWorkerChunk`, in chunk order. The pool is the sole parse
-                // path — `getOrCreateWorkerPool` returns a pool or throws.
-                const dispatchPromise = dispatchChunkParse(chunkFiles, activeWorkerPool, progressForChunk, undefined, chunkHash ?? undefined);
-                // Mark handled so a rejection during the overlap drain below isn't
-                // flagged as unhandled; the `await` re-throws it for real handling.
-                dispatchPromise.catch(() => { });
-                if (pendingWorkerChunk) {
-                    await finalizeWorkerChunk(pendingWorkerChunk);
-                    pendingWorkerChunk = null;
-                }
-                let chunkResults;
-                try {
-                    chunkResults = await dispatchPromise;
-                }
-                catch (err) {
-                    if (!(err instanceof WorkerPoolInitializationError))
-                        throw err;
-                    // Every worker crashed during startup and the pool's bounded
-                    // self-heal was exhausted. Fail fast (#1741) — there is no sequential
-                    // parser to degrade to. `handleWorkerStartupFailure` always throws, so
-                    // `chunkResults` stays definitely assigned for the parked chunk below.
-                    handleWorkerStartupFailure(err);
-                }
-                pendingWorkerChunk = {
-                    rawResults: chunkResults,
-                    chunkIdx,
-                    chunkHash,
-                    chunkFiles,
-                    chunkStartMs,
-                };
+                roundEntries.push({ kind: 'miss', chunkIdx, chunkHash, chunkFiles, chunkStartMs });
+                roundIsFull = roundBudget.addChunk(chunkFiles.map((file) => file.content));
+                queuedFilesSoFar += chunkFiles.length;
             }
+            // One cap, on what the main thread is holding. That bounds the worker
+            // round too, since a round's dispatched bytes are a subset of its
+            // buffered bytes.
+            if (roundIsFull)
+                await closeRound();
             // (Per-chunk aggregation + parse-cache write + throughput log now run in
             // `applyChunkResults` / `finalizeWorkerChunk` — see the merge-pipelining
             // block above. Route/import/inheritance edges are emitted later: route
             // resolution in the single end-of-loop pass below, the rest by the
             // scope-resolution phase, RING4-2 #943.)
         }
-        // Drain the final parked worker chunk — the last pipelined chunk has no
-        // successor to overlap its merge with, so merge + finalize it here.
-        if (pendingWorkerChunk) {
-            await finalizeWorkerChunk(pendingWorkerChunk);
-            pendingWorkerChunk = null;
+        // Drain the tail: close the partially-filled round, then drain the round
+        // it parked — the last round has no successor to overlap its merge with.
+        if (roundEntries.length > 0)
+            await closeRound();
+        if (pendingRound) {
+            const last = pendingRound;
+            pendingRound = null;
+            await drainRound(last);
         }
         if (isDev && parseCache && (chunkCacheHits > 0 || chunkCacheMisses > 0)) {
             logger.info(`📦 parse-cache summary: ${chunkCacheHits} chunk hit(s), ${chunkCacheMisses} miss(es) across ${numChunks} chunk(s)`);

package/dist/core/ingestion/pipeline-phases/parse-round-budget.d.ts

@@ -0,0 +1,42 @@
+/**
+ * The fold that decides when an open dispatch round closes.
+ *
+ * Extracted so the decision is a shared, inspectable unit rather than four
+ * loose statements inside `runChunkedParseAndResolve`. The parse loop is
+ * STREAMING — it reads chunk contents lazily, so it cannot know every chunk's
+ * size up front and cannot "plan" rounds ahead. That makes an accumulator, not
+ * a planner, the honest shape: feed it each chunk as it is queued and it tells
+ * you whether the round is now full.
+ *
+ * Being a real unit is what makes round cadence observable. Round boundaries
+ * are otherwise invisible from outside the parse phase: they change no graph
+ * output (that is the point of batching) and surface only in a log line, which
+ * is why `bench/parse-dispatch-rounds` measures this directly rather than
+ * inferring cadence from a full analyze.
+ */
+/** Bytes a file contributes to the open round's retained total. */
+export declare const roundFileBytes: (content: string) => number;
+export interface RoundBudget {
+    /**
+     * Add one queued chunk's files. Returns true when the round is now full and
+     * the caller should close it. Closing resets the accumulator.
+     */
+    addChunk(contents: readonly string[]): boolean;
+    /** Bytes currently held by the open round. */
+    readonly bufferedBytes: number;
+    /** Reset without closing — used when the caller closes for another reason. */
+    reset(): void;
+}
+/**
+ * `budgetBytes` bounds what the main thread HOLDS, counting cache hits as well
+ * as misses. Counting only cache-missing bytes would bound just the work sent
+ * to workers, so a warm run — where nothing misses — would never reach the
+ * close condition and would buffer every chunk's cached output until the tail
+ * drain. That is the #2649 heap failure on a large repo.
+ *
+ * Measured in UTF-8 bytes, matching `estimateItemBytes` in the worker pool.
+ * `String.length` would return UTF-16 code units, undercounting non-ASCII
+ * source by up to 3x and letting a CJK-heavy repo hold well past its nominal
+ * budget before draining.
+ */
+export declare const createRoundBudget: (budgetBytes: number) => RoundBudget;

package/dist/core/ingestion/pipeline-phases/parse-round-budget.js

@@ -0,0 +1,50 @@
+/**
+ * The fold that decides when an open dispatch round closes.
+ *
+ * Extracted so the decision is a shared, inspectable unit rather than four
+ * loose statements inside `runChunkedParseAndResolve`. The parse loop is
+ * STREAMING — it reads chunk contents lazily, so it cannot know every chunk's
+ * size up front and cannot "plan" rounds ahead. That makes an accumulator, not
+ * a planner, the honest shape: feed it each chunk as it is queued and it tells
+ * you whether the round is now full.
+ *
+ * Being a real unit is what makes round cadence observable. Round boundaries
+ * are otherwise invisible from outside the parse phase: they change no graph
+ * output (that is the point of batching) and surface only in a log line, which
+ * is why `bench/parse-dispatch-rounds` measures this directly rather than
+ * inferring cadence from a full analyze.
+ */
+/** Bytes a file contributes to the open round's retained total. */
+export const roundFileBytes = (content) => Buffer.byteLength(content, 'utf8');
+/**
+ * `budgetBytes` bounds what the main thread HOLDS, counting cache hits as well
+ * as misses. Counting only cache-missing bytes would bound just the work sent
+ * to workers, so a warm run — where nothing misses — would never reach the
+ * close condition and would buffer every chunk's cached output until the tail
+ * drain. That is the #2649 heap failure on a large repo.
+ *
+ * Measured in UTF-8 bytes, matching `estimateItemBytes` in the worker pool.
+ * `String.length` would return UTF-16 code units, undercounting non-ASCII
+ * source by up to 3x and letting a CJK-heavy repo hold well past its nominal
+ * budget before draining.
+ */
+export const createRoundBudget = (budgetBytes) => {
+    let bufferedBytes = 0;
+    return {
+        addChunk(contents) {
+            for (const content of contents)
+                bufferedBytes += roundFileBytes(content);
+            if (bufferedBytes >= budgetBytes) {
+                bufferedBytes = 0;
+                return true;
+            }
+            return false;
+        },
+        get bufferedBytes() {
+            return bufferedBytes;
+        },
+        reset() {
+            bufferedBytes = 0;
+        },
+    };
+};

package/dist/core/ingestion/scope-resolution/contract/scope-resolver.d.ts

@@ -615,6 +615,19 @@ export interface ScopeResolver {
     readonly populateWorkspaceOwners?: (parsedFiles: readonly ParsedFile[], ctx: {
         readonly fileContents: ReadonlyMap<string, string>;
     }) => void;
+    /**
+     * Optional workspace-wide enrichment of extracted reference sites. Runs
+     * after all files have been extracted and before reference finalization.
+     * Use this when a per-file capture needs conservative facts from an
+     * imported sibling (for example a compile-time branch constant).
+     */
+    readonly populateWorkspaceReferences?: (parsedFiles: ParsedFile[], ctx: {
+        readonly fileContents: ReadonlyMap<string, string>;
+        readonly treeCache?: {
+            get(filePath: string): unknown;
+        };
+        readonly resolutionConfig?: unknown;
+    }) => void;
     /**
      * Recognize a `super(...)`-style receiver text. Python returns
      * `/^super\s*\(/.test(t)`. Java returns `t === 'super'`. C++ may

package/dist/core/ingestion/scope-resolution/pipeline/phase.js

@@ -55,6 +55,7 @@ const NOOP_OUTPUT = Object.freeze({
 /** Select source files that must be materialized for one resolver pass. */
 export function selectScopeSourcePathsToRead(provider, primaryFilePaths, preExtractedByPath) {
     const hasPostExtractHooks = provider.populateWorkspaceOwners !== undefined ||
+        provider.populateWorkspaceReferences !== undefined ||
         provider.populateNamespaceSiblings !== undefined ||
         provider.populateRangeBindings !== undefined ||
         provider.emitPostResolutionEdges !== undefined;

package/dist/core/ingestion/scope-resolution/pipeline/run.js

@@ -312,6 +312,11 @@ export function runScopeResolution(input, provider) {
     }
     logHeapProbe('sr-extract-end', `lang=${provider.language} parsedFiles=${parsedFiles.length} preExtractedHits=${preExtractedHits} skipped=${filesSkipped}`);
     provider.populateWorkspaceOwners?.(parsedFiles, { fileContents: getFileContents() });
+    provider.populateWorkspaceReferences?.(parsedFiles, {
+        fileContents: getFileContents(),
+        treeCache,
+        resolutionConfig: input.resolutionConfig,
+    });
     // A callable-flow-only provider has no reason to build the whole-graph
     // lookup or finalize ordinary references when none of its files emitted a
     // callable fact. This keeps the opt-in path proportional to source scanning