gitnexus 1.6.6-rc.98 → 1.6.6

package/dist/core/ingestion/workers/worker-pool.js

@@ -79,10 +79,29 @@ export class WorkerPoolDispatchError extends Error {
 }
 export class WorkerPoolInitializationError extends WorkerPoolDispatchError {
     readinessFailures;
-    constructor(message, quarantinedPaths = [], readinessFailures = []) {
+    /** Pool's automatic classification of the startup crash (#1741). */
+    crashClass;
+    constructor(message, quarantinedPaths = [], readinessFailures = [], crashClass = 'transient-exhausted') {
         super(message, quarantinedPaths);
         this.name = 'WorkerPoolInitializationError';
         this.readinessFailures = readinessFailures;
+        this.crashClass = crashClass;
+    }
+}
+/**
+ * Thrown when a caller asks GitNexus to parse without the worker pool —
+ * `--workers 0`, `GITNEXUS_WORKER_POOL_SIZE=0`, or `skipWorkers: true`.
+ *
+ * GitNexus no longer has a sequential parser: the worker pool (with its
+ * quarantine + respawn/recycle + circuit-breaker resilience) is the SOLE
+ * parse path. These channels used to select an in-process fallback; they are
+ * now hard configuration errors so the operator gets an actionable message
+ * instead of silently parsing through a (deleted) slower path.
+ */
+export class WorkerPoolDisabledError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'WorkerPoolDisabledError';
     }
 }
 /**
@@ -117,6 +136,89 @@ const WORKER_READY_TIMEOUT_MS = 5_000;
  * `GITNEXUS_WORKER_POOL_SIZE` or `--workers <N>`.
  */
 const DEFAULT_POOL_SIZE_CAP = 16;
+// ── Self-healing startup restart policy (#1741) ──────────────────────────────
+// A worker that crashes during top-of-script init (broken native binding, bad
+// import) is retried a BOUNDED number of times with jittered backoff before
+// its slot is dropped, so a transient blip self-heals with no operator
+// intervention. The bound is the whole point of #1741: recovery must never
+// become a silent, unbounded "stuck" run. When the budget is exhausted (or a
+// deterministic crash-loop is detected), the slot is dropped; if every slot is
+// dropped the first dispatch fails fast with the captured cause.
+/** Retries beyond the first attempt, per slot, to bring a startup worker ready. */
+const STARTUP_RESTART_BUDGET = 2;
+const RESTART_BACKOFF_BASE_MS = 250;
+const RESTART_BACKOFF_CAP_MS = 2_000;
+/**
+ * When this many freshly-spawned workers crash with the SAME crash signature
+ * before ANY worker reaches the `{type:'ready'}` handshake, the failure is
+ * deterministic (the #1741 missing-binding case: every worker prints a
+ * byte-identical native-binding stack). The pool stops retrying immediately
+ * instead of burning every slot's budget, and fails fast with the cause.
+ */
+const DETERMINISTIC_STARTUP_FINGERPRINT_THRESHOLD = 2;
+/**
+ * Capped exponential backoff with FULL jitter (AWS "Exponential Backoff And
+ * Jitter"): random(0, min(CAP, BASE·2^attempt)). Full jitter de-synchronizes
+ * the N workers that crash near-simultaneously on a shared startup fault so
+ * their respawns don't re-storm in lockstep (Google SRE thundering herd).
+ */
+function startupBackoffMs(attempt) {
+    const ceil = Math.min(RESTART_BACKOFF_CAP_MS, RESTART_BACKOFF_BASE_MS * 2 ** attempt);
+    return Math.floor(Math.random() * (ceil + 1));
+}
+/**
+ * Sleep used between startup self-heal retries. The timer is intentionally NOT
+ * `unref`'d: a pending retry is necessary work, so it must keep the event loop
+ * alive long enough to actually respawn — otherwise a pool whose only live work
+ * is a startup backoff could let the process exit mid-recovery (#1741). To
+ * avoid wedging shutdown, the timer registers a cancel function in `pending`;
+ * `terminate()` invokes those cancels to `clearTimeout` and resolve early, and
+ * a normally-fired timer removes its own cancel. `aborted()` is checked once up
+ * front; the CALLER re-checks after wake (it owns the terminated/deterministic
+ * decision) — this function does not itself re-evaluate abort on wake.
+ */
+function abortableSleep(ms, aborted, pending) {
+    return new Promise((resolve) => {
+        if (ms <= 0 || aborted()) {
+            resolve();
+            return;
+        }
+        // `cancel` is registered so terminate() can clear a pending backoff; it is
+        // also the timer's own callback, so a normally-fired sleep self-deregisters.
+        const cancel = () => {
+            clearTimeout(timer);
+            pending.delete(cancel);
+            resolve();
+        };
+        const timer = setTimeout(cancel, ms);
+        pending.add(cancel);
+    });
+}
+/**
+ * Normalize a worker crash message into a stable signature so two instances of
+ * the SAME deterministic crash compare equal while unrelated crashes don't.
+ * Strips hex addresses, digit runs (pids / line numbers / timestamps) and
+ * absolute paths. Best-effort by design: the deterministic classification's
+ * correctness rests on the STRUCTURAL signal (zero workers ever ready + startup
+ * budget exhausted), so an imperfect signature only changes how fast the
+ * short-circuit fires, never whether the pool ultimately fails fast. Even a
+ * stderr-less crash normalizes its "exited with code N" message to a stable
+ * key, so the empty-stderr timing case still groups.
+ *
+ * @internal Exported for unit tests; production callers are in this module.
+ */
+export function crashSignature(message) {
+    return (message
+        .replace(/0x[0-9a-fA-F]+/g, '0xADDR') // 0x-prefixed addresses
+        // Windows backslash paths (optional drive letter), e.g. C:\Users\ci\Temp\w-7f3a.js
+        .replace(/(?:[A-Za-z]:)?(?:\\[^\s\\'"]+)+/g, '\\PATH')
+        .replace(/(?:\/[^\s:'"]+)+/g, '/PATH') // POSIX paths
+        .replace(/\b[0-9a-fA-F]{6,}\b/g, 'HEX') // bare hex runs (ASLR addrs / backtrace tokens)
+        .replace(/[0-9]+/g, 'N') // pids / line numbers / exit codes / timestamps
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, 300));
+}
 function positiveInteger(value) {
     const parsed = typeof value === 'string' ? Number(value) : value;
     return typeof parsed === 'number' && Number.isFinite(parsed) && parsed > 0
@@ -152,12 +254,37 @@ export function resolveWorkerPoolOptions(options = {}, poolSize) {
             Math.max(DEFAULT_CONSECUTIVE_FAILURE_THRESHOLD_FLOOR, poolSize ?? 0),
     };
 }
+/**
+ * The pool size requested via the `GITNEXUS_WORKER_POOL_SIZE` env var, or
+ * `undefined` when unset, empty/whitespace, or invalid. Module-internal sizing
+ * reader consumed by {@link resolveAutoPoolSize} (the env override) and
+ * {@link workerPoolDisabledByEnv} (the disabled-channel check). Reads only —
+ * never mutates `process.env`. Empty/whitespace is treated as *unset* (falls
+ * through to the auto formula), not as 0 — an empty assignment (`export
+ * GITNEXUS_WORKER_POOL_SIZE=`) is an accident, not a request for zero workers;
+ * only a literal `0` disables the pool.
+ */
+function envWorkerPoolSize() {
+    const raw = process.env.GITNEXUS_WORKER_POOL_SIZE;
+    if (raw === undefined || raw.trim() === '')
+        return undefined;
+    return nonNegativeInteger(raw);
+}
+/**
+ * True when the operator set `GITNEXUS_WORKER_POOL_SIZE=0` — the env-channel
+ * equivalent of `--workers 0`. The parse phase consults this (only when no
+ * explicit `--workers <N>` was passed) and HARD-ERRORS: sequential parsing was
+ * removed, so a disabled pool is an actionable configuration error, not a
+ * silent fallback. An explicit positive `--workers N` always wins.
+ */
+export function workerPoolDisabledByEnv() {
+    return envWorkerPoolSize() === 0;
+}
 /**
  * Resolve the auto-default worker pool size when no explicit `poolSize`
  * arg is passed to `createWorkerPool`. Precedence:
  *
- * 1. `GITNEXUS_WORKER_POOL_SIZE` env var (operator override; set by
- *    `--workers <N>` on the CLI).
+ * 1. `GITNEXUS_WORKER_POOL_SIZE` env var (operator override).
  * 2. `os.cpus().length - 1`, clamped to `[1, DEFAULT_POOL_SIZE_CAP]`.
  *
  * The cap exists because past ~16 workers the main-thread merge /
@@ -170,7 +297,7 @@ export function resolveWorkerPoolOptions(options = {}, poolSize) {
  * on the env / default.
  */
 export function resolveAutoPoolSize() {
-    const envOverride = nonNegativeInteger(process.env.GITNEXUS_WORKER_POOL_SIZE);
+    const envOverride = envWorkerPoolSize();
     if (envOverride !== undefined)
         return envOverride;
     // Prefer os.availableParallelism (Node 18.14+) so cgroup CPU limits
@@ -184,6 +311,69 @@ export function resolveAutoPoolSize() {
     const cores = typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length;
     return Math.min(DEFAULT_POOL_SIZE_CAP, Math.max(1, cores - 1));
 }
+/**
+ * Max characters of a worker's stderr retained for crash diagnostics. A
+ * native-binding load failure or a top-of-script throw prints a stack to
+ * stderr; we keep the tail so `waitForWorkerReady` can attach the real
+ * reason to its rejection instead of the generic "did not report ready".
+ */
+const WORKER_STDERR_TAIL_LIMIT = 4000;
+/**
+ * Per-worker captured stderr tail. Populated only for workers spawned with
+ * `{ stderr: true }` (the production factory below). Test-injected workers
+ * via `workerFactory` typically inherit the parent's stderr and have no
+ * `worker.stderr` stream — those are simply skipped (empty tail). A WeakMap
+ * so the buffer is released when the worker is GC'd.
+ */
+const workerStderrTails = new WeakMap();
+/**
+ * Tee a worker's stderr into a bounded in-memory tail (for surfacing the
+ * real crash on a startup failure) while still mirroring it to the parent
+ * process's stderr — preserving the live-diagnostics behavior workers had
+ * when they inherited stderr, before `{ stderr: true }` redirected it to a
+ * stream. No-op when the worker has no `stderr` stream (test factories).
+ */
+function captureWorkerStderr(worker) {
+    const stream = worker.stderr;
+    if (!stream)
+        return;
+    const buf = { text: '' };
+    workerStderrTails.set(worker, buf);
+    stream.on('data', (chunk) => {
+        const s = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
+        process.stderr.write(s);
+        buf.text = (buf.text + s).slice(-WORKER_STDERR_TAIL_LIMIT);
+    });
+    // A stderr stream error must never crash the pool.
+    stream.on('error', () => undefined);
+}
+/** Captured stderr tail for a worker, trimmed; '' when nothing was captured. */
+function workerStderrTail(worker) {
+    return workerStderrTails.get(worker)?.text.trim() ?? '';
+}
+/** Append the worker's captured stderr to a readiness-failure message. */
+function withStderr(worker, message) {
+    const tail = workerStderrTail(worker);
+    return tail ? `${message}. Worker stderr:\n${tail}` : message;
+}
+/**
+ * Build a worker-death reason string that carries the worker-side stack when one
+ * is available (#2068). The stack is appended AFTER the `Worker N error: <msg>`
+ * prefix so every prefix/substring consumer downstream — recoverAndResume →
+ * handleWorkerDeath → the circuit-breaker `WorkerPoolDispatchError` message, and
+ * the tests that regex-match those — keeps working unchanged, while the operator
+ * now gets the real frame instead of a bare one-liner. The stack's first line is
+ * normally the message itself; keeping both is harmless and the indented block
+ * scans cleanly in a log. The stack is capped at WORKER_STDERR_TAIL_LIMIT,
+ * mirroring the sibling stderr-tail bound, so a pathological error type (or a
+ * raised `Error.stackTraceLimit`) can't bloat the death reason. `stack` is
+ * `undefined` for an older worker build (or a thrown non-Error), in which case
+ * the reason is exactly the prior message-only form.
+ */
+function workerErrorReason(workerIndex, message, stack) {
+    const base = `Worker ${workerIndex} error: ${message}`;
+    return stack ? `${base}\n  worker stack:\n${stack.slice(0, WORKER_STDERR_TAIL_LIMIT)}` : base;
+}
 /**
  * Wait for a freshly-spawned replacement worker to emit the
  * `{type:'ready'}` handshake from `parse-worker.ts` before treating its
@@ -220,22 +410,24 @@ function waitForWorkerReady(worker) {
         };
         const onError = (err) => {
             cleanup();
-            reject(err);
+            // The 'error' event carries the real top-of-script exception; enrich it
+            // with the worker's stderr tail (native-binding stacks land there).
+            reject(new Error(withStderr(worker, err.message)));
         };
         const onExit = (code) => {
             cleanup();
-            reject(new Error(`Replacement worker exited with code ${code} before reporting ready`));
+            reject(new Error(withStderr(worker, `Replacement worker exited with code ${code} before reporting ready`)));
         };
         const onMessageError = (err) => {
             cleanup();
-            reject(new Error(`Replacement worker emitted messageerror before reporting ready: ${err.message}`));
+            reject(new Error(withStderr(worker, `Replacement worker emitted messageerror before reporting ready: ${err.message}`)));
         };
         // `timer` is declared after `cleanup` so the cleanup closure can reference
         // it. The const is reached before any handler attaches below, so no TDZ
         // access can fire from the listeners.
         const timer = setTimeout(() => {
             cleanup();
-            reject(new Error(`Replacement worker did not report ready within ${WORKER_READY_TIMEOUT_MS}ms — likely crashed during top-of-script init`));
+            reject(new Error(withStderr(worker, `Replacement worker did not report ready within ${WORKER_READY_TIMEOUT_MS}ms — likely crashed during top-of-script init`)));
         }, WORKER_READY_TIMEOUT_MS);
         worker.on('message', onMessage);
         worker.once('error', onError);
@@ -275,7 +467,7 @@ function inFlightExcludePath(job, lastProgress) {
     const path = itemPath(job.items[lastProgress]);
     return path ? [path] : [];
 }
-function createJobs(items, maxItems, maxBytes, timeoutMs) {
+function createJobs(items, maxItems, maxBytes, timeoutMs, chunkHash) {
     const jobs = [];
     let startIndex = 0;
     let batch = [];
@@ -289,6 +481,7 @@ function createJobs(items, maxItems, maxBytes, timeoutMs) {
             estimatedBytes: batchBytes,
             attempt: 0,
             splitDepth: 0,
+            chunkHash,
             timeoutMs,
             cumulativeTimeoutMs: timeoutMs,
         });
@@ -340,7 +533,31 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
     }
     const size = poolSize ?? resolveAutoPoolSize();
     const poolOptions = resolveWorkerPoolOptions(options, size);
-    const spawnWorker = options?.workerFactory ?? ((url) => new Worker(url));
+    // Production factory spawns with `{ stderr: true }` so a worker's crash
+    // output is redirected to a `worker.stderr` stream we can tee + capture
+    // (see captureWorkerStderr) and attach to readiness-failure messages —
+    // instead of the generic "did not report ready" that hid the real cause
+    // in #1741. Test factories (workerFactory) are used verbatim.
+    // Bake the (immutable) ParsedFile store path into the factory closure so it
+    // reaches EVERY spawned worker — including respawns, which reuse this same
+    // factory — via `workerData`, read once at worker init. The `(url) => Worker`
+    // signature is unchanged so the zero-arg test factories keep working.
+    const parsedFileStoreStoragePath = options?.parsedFileStoreStoragePath;
+    const durableParsedFileStoragePath = options?.durableParsedFileStoragePath;
+    const workerStoreData = parsedFileStoreStoragePath || durableParsedFileStoragePath
+        ? { parsedFileStoreStoragePath, durableParsedFileStoragePath }
+        : undefined;
+    const spawnWorker = options?.workerFactory ??
+        ((url) => new Worker(url, {
+            stderr: true,
+            workerData: workerStoreData,
+        }));
+    /** Spawn + wire stderr capture in one step (used by all spawn sites). */
+    const spawnAndCapture = (url) => {
+        const worker = spawnWorker(url);
+        captureWorkerStderr(worker);
+        return worker;
+    };
     const workers = new Array(size);
     const retiredWorkers = new Set();
     const respawnCount = new Array(size).fill(0);
@@ -369,6 +586,9 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
     const slotGenerations = new Array(size).fill(0);
     let poolBroken = false;
     let poolFailure;
+    // Set by `terminate()` (below). Also read by the self-healing startup loop so
+    // a terminate during startup aborts pending backoff/retries (#1741).
+    let terminated = false;
     const terminateTrackedWorkers = async (liveWorkers) => {
         const retired = Array.from(retiredWorkers);
         await Promise.all([
@@ -378,42 +598,97 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
         retiredWorkers.clear();
     };
     for (let i = 0; i < size; i++) {
-        workers[i] = spawnWorker(workerUrl);
+        workers[i] = spawnAndCapture(workerUrl);
         activeSlots.add(i);
     }
-    // Symmetrize the readiness gate across initial and replacement spawn
-    // paths. `replaceWorker` already awaits `waitForWorkerReady` per
-    // replacement so an init-crashing worker is dropped before dispatch
-    // sees it. The initial-spawn loop above didn't — a worker whose
-    // top-of-script init crashes (failed tree-sitter native binding,
-    // missing dependency) would only be noticed at the first dispatch's
-    // 30s idle timeout, vs the 5s WORKER_READY_TIMEOUT_MS bound that
-    // replacements enjoy.
+    // ── Self-healing startup readiness (#1741) ────────────────────────────────
+    // Bring every initial slot to readiness with a BOUNDED, jittered retry loop
+    // instead of dropping it on the first crash. This symmetrizes the gate with
+    // the runtime `replaceWorker` path (which already respawns a crashed slot),
+    // and adds genuine self-healing at startup:
     //
-    // The promise below settles every initial slot in parallel and drops
-    // unready slots from `activeSlots` before any dispatch can fire.
-    // `dispatch` awaits it via `initialReadyGate` on first invocation.
-    // Wrapped in a single `Promise.allSettled` so a slow worker doesn't
-    // block ready workers from being usable — first dispatch waits for
-    // all slots' verdicts (good or bad).
-    const initialReadyGate = Promise.allSettled(workers.map(async (w, i) => {
-        if (!w)
-            return;
-        try {
-            await waitForWorkerReady(w);
-        }
-        catch (err) {
-            initialReadinessFailures.push(err instanceof Error ? err.message : String(err));
-            logger.warn({
-                workerIndex: i,
-                err: err instanceof Error ? err.message : String(err),
-            }, `Worker ${i} did not report ready on initial spawn; dropping slot.`);
-            await w.terminate().catch(() => undefined);
-            workers[i] = undefined;
-            activeSlots.delete(i);
+    //  - TRANSIENT crash (a one-off OS hiccup / fork throttle): the slot is
+    //    respawned after jittered backoff and retried, up to STARTUP_RESTART_BUDGET
+    //    — so a blip heals itself with no operator intervention.
+    //  - DETERMINISTIC crash-loop (every worker dies with the SAME signature
+    //    before any reaches ready — the #1741 missing-binding case): detected via
+    //    `crashSignature` and short-circuited immediately, so the pool gives up in
+    //    ~1s rather than burning every slot's budget.
+    //
+    // When the loop exhausts, the slot is dropped from `activeSlots`. If EVERY
+    // slot is dropped, the first dispatch throws WorkerPoolInitializationError
+    // carrying the captured crash cause + classification — never a silent hang.
+    // Correctness of the deterministic short-circuit rests on the STRUCTURAL
+    // signal (zero workers ever ready + budget exhausted), not on signature
+    // matching alone: a missed match only costs a few seconds of extra retrying.
+    // Deterministic crash-loop detection (#1741). A crash counts toward
+    // "deterministic" ONLY after its signature reproduces across a respawn on the
+    // same slot — so every slot is guaranteed at least one self-heal attempt and
+    // a simultaneous attempt-0 crash storm (e.g. transient `spawn EAGAIN` under
+    // fork pressure) cannot be misclassified as deterministic. We short-circuit
+    // once enough DISTINCT slots have each reproduced: ≥2 normally, or 1 for a
+    // size-1 pool. Until then the structural floor (every slot exhausts its
+    // budget) still bounds the worst case, so a missed match only costs retries.
+    const lastStartupSignature = new Map();
+    const reproducedStartupSlots = new Set();
+    const deterministicSlotThreshold = Math.min(DETERMINISTIC_STARTUP_FINGERPRINT_THRESHOLD, size);
+    let deterministicStartupDetected = false;
+    let anyWorkerReachedReady = false;
+    // Cancel functions for in-flight startup backoffs (see abortableSleep). The
+    // backoff timer is ref'd so a retry actually runs; terminate() invokes these
+    // to clear pending backoffs and resolve their sleeps so the slot loops wake,
+    // see `terminated`, and give up — instead of the process staying pinned for
+    // the backoff cap after terminate (#1741).
+    const pendingStartupTimers = new Set();
+    const bringSlotReady = async (i) => {
+        for (let attempt = 0;; attempt++) {
+            const worker = workers[i];
+            if (!worker)
+                return; // terminated mid-startup
+            try {
+                await waitForWorkerReady(worker);
+                anyWorkerReachedReady = true;
+                return; // ready — slot stays in activeSlots
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                const sig = crashSignature(msg);
+                // Same signature as this slot's previous attempt => it survived a
+                // respawn, so retrying this slot is futile. (First crash has no prior
+                // signature, so attempt 0 never counts — every slot self-heals once.)
+                if (lastStartupSignature.get(i) === sig)
+                    reproducedStartupSlots.add(i);
+                lastStartupSignature.set(i, sig);
+                if (!anyWorkerReachedReady && reproducedStartupSlots.size >= deterministicSlotThreshold) {
+                    deterministicStartupDetected = true;
+                }
+                await worker.terminate().catch(() => undefined);
+                workers[i] = undefined;
+                const giveUp = terminated || deterministicStartupDetected || attempt >= STARTUP_RESTART_BUDGET;
+                if (giveUp) {
+                    initialReadinessFailures.push(msg);
+                    activeSlots.delete(i);
+                    logger.warn({ workerIndex: i, attempt, err: msg, deterministic: deterministicStartupDetected }, deterministicStartupDetected
+                        ? `Worker ${i} hit a deterministic startup crash-loop; dropping slot without further retries.`
+                        : `Worker ${i} did not report ready after ${attempt + 1} attempt(s); dropping slot.`);
+                    return;
+                }
+                // Transient: jittered backoff, then respawn the slot and retry.
+                await abortableSleep(startupBackoffMs(attempt), () => terminated || deterministicStartupDetected, pendingStartupTimers);
+                if (terminated || deterministicStartupDetected) {
+                    initialReadinessFailures.push(msg);
+                    activeSlots.delete(i);
+                    return;
+                }
+                logger.warn({ workerIndex: i, attempt: attempt + 1 }, `Worker ${i} crashed during startup; respawning slot (self-heal attempt ${attempt + 1}/${STARTUP_RESTART_BUDGET}).`);
+                workers[i] = spawnAndCapture(workerUrl);
+            }
         }
-    })).then(() => undefined);
-    const dispatch = async (items, onProgress) => {
+    };
+    // First dispatch awaits this; it settles every slot's bounded retry loop in
+    // parallel and drops the unrecoverable ones before any dispatch can fire.
+    const initialReadyGate = Promise.allSettled(workers.map((_, i) => bringSlotReady(i))).then(() => undefined);
+    const dispatch = async (items, onProgress, chunkHash) => {
         // Await the initial-spawn readiness gate (F13). On first dispatch
         // this blocks for up to WORKER_READY_TIMEOUT_MS while every initial
         // worker's `{type:'ready'}` handshake is checked; on subsequent
@@ -434,7 +709,13 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
             const detail = initialReadinessFailures.length > 0
                 ? ` after initial ready handshake: ${initialReadinessFailures.join('; ')}`
                 : '';
-            throw new WorkerPoolInitializationError(`Worker pool has no active workers${detail}`, [], initialReadinessFailures);
+            // The bounded self-heal exhausted (or short-circuited a deterministic
+            // crash-loop). Classify automatically so the caller renders the real
+            // cause without consulting any operator flag (#1741).
+            const crashClass = deterministicStartupDetected
+                ? 'deterministic-startup'
+                : 'transient-exhausted';
+            throw new WorkerPoolInitializationError(`Worker pool has no active workers${detail}`, [], initialReadinessFailures, crashClass);
         }
         // Layer 3: filter out quarantined paths so a known-bad file never reaches
         // a worker again this pool lifetime. The caller queries
@@ -448,7 +729,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
         }
         if (dispatchableItems.length === 0)
             return [];
-        const jobs = createJobs(dispatchableItems, poolOptions.subBatchSize, poolOptions.subBatchMaxBytes, poolOptions.subBatchIdleTimeoutMs);
+        const jobs = createJobs(dispatchableItems, poolOptions.subBatchSize, poolOptions.subBatchMaxBytes, poolOptions.subBatchIdleTimeoutMs, chunkHash);
         return new Promise((resolve, reject) => {
             const results = [];
             const inFlightProgress = new Array(size).fill(0);
@@ -549,7 +830,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                 await removeWorkerFromSlot(workerIndex, mode, reason);
                 if (stopped)
                     return false;
-                const replacement = spawnWorker(workerUrl);
+                const replacement = spawnAndCapture(workerUrl);
                 try {
                     await waitForWorkerReady(replacement);
                 }
@@ -663,6 +944,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                     estimatedBytes: filtered.reduce((sum, item) => sum + estimateItemBytes(item), 0),
                     attempt: job.attempt,
                     splitDepth: job.splitDepth,
+                    chunkHash: job.chunkHash,
                     timeoutMs: job.timeoutMs,
                     cumulativeTimeoutMs: job.cumulativeTimeoutMs,
                 });
@@ -753,6 +1035,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                         estimatedBytes: firstItems.reduce((sum, item) => sum + estimateItemBytes(item), 0),
                         attempt: job.attempt,
                         splitDepth: job.splitDepth + 1,
+                        chunkHash: job.chunkHash,
                         timeoutMs: nextTimeout,
                         cumulativeTimeoutMs: nextCumulative,
                     };
@@ -762,6 +1045,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                         estimatedBytes: secondItems.reduce((sum, item) => sum + estimateItemBytes(item), 0),
                         attempt: job.attempt,
                         splitDepth: job.splitDepth + 1,
+                        chunkHash: job.chunkHash,
                         timeoutMs: nextTimeout,
                         cumulativeTimeoutMs: nextCumulative,
                     };
@@ -1064,12 +1348,15 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                     else if (msg.type === 'sub-batch-done') {
                         waitingForFlush = true;
                         resetIdleTimer();
-                        worker.postMessage({ type: 'flush' });
+                        // Carry the chunk hash on the flush so the worker can write a
+                        // durable, content-addressed ParsedFile shard (warm-cache reuse)
+                        // at the flush boundary where `accumulated.parsedFiles` is complete.
+                        worker.postMessage({ type: 'flush', chunkHash: job.chunkHash });
                     }
                     else if (msg.type === 'error') {
                         settled = true;
                         cleanup();
-                        void recoverAndResume(`Worker ${workerIndex} error: ${msg.error}`, resolveExcludePaths());
+                        void recoverAndResume(workerErrorReason(workerIndex, msg.error, msg.errorStack), resolveExcludePaths());
                     }
                     else if (msg.type === 'result') {
                         if (!waitingForFlush) {
@@ -1110,7 +1397,12 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                     if (!settled) {
                         settled = true;
                         cleanup();
-                        void recoverAndResume(`Worker ${workerIndex} error: ${err.message}`, resolveExcludePaths());
+                        // The Node 'error' event fires on an UNCAUGHT worker throw (one that
+                        // escaped the worker's own try/catch, or an async rejection). Unlike
+                        // the `{type:'error'}` message, the event delivers a real Error whose
+                        // `.stack` is the worker-side frame — carry it so the surfaced reason
+                        // points at the actual failure site, not just `err.message` (#2068).
+                        void recoverAndResume(workerErrorReason(workerIndex, err.message, err.stack), resolveExcludePaths());
                     }
                 };
                 const exitHandler = (code) => {
@@ -1165,9 +1457,13 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
                 runWorker(slotIndex);
         });
     };
-    let terminated = false;
     const terminate = async () => {
         terminated = true;
+        // Cancel any in-flight startup backoff so its ref'd timer doesn't keep the
+        // event loop alive after terminate; each cancel resolves the awaiting sleep
+        // and the slot loop then sees `terminated` and gives up (#1741).
+        for (const cancel of [...pendingStartupTimers])
+            cancel();
         // `.catch(() => undefined)` per-worker matches every other terminate
         // site in this file. Without it, a hung/OOM-killed worker's terminate
         // rejection escapes `Promise.all` and replaces the original pipeline
@@ -1190,6 +1486,7 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
             quarantined: quarantine.size,
             poolBroken,
             terminated,
+            pendingStartupTimers: pendingStartupTimers.size,
             slotGenerations: slotGenerations.slice(),
         }),
     };

package/dist/core/lbug/csv-generator.js

@@ -14,6 +14,21 @@
 import fs from 'fs/promises';
 import { createWriteStream } from 'fs';
 import path from 'path';
+import { parseTruthyEnv } from '../ingestion/utils/env.js';
+/**
+ * Deterministic output ordering — optional (out-of-core / windowed-resolve
+ * enabler). When `GITNEXUS_SORT_GRAPH_OUTPUT` is set, nodes and relationships
+ * are emitted sorted by their (unique, dedup-key) graph `id` rather than in
+ * graph-insertion order, making the CSV a pure function of the graph's node/edge
+ * SET instead of of emit order. Default off returns the iterator untouched, so
+ * the bytes are identical to today. With it on, a windowed/out-of-core emit
+ * (the later windowed-resolve work) need only reproduce the same edge SET, not the global insertion order —
+ * which removes "CSV row order == Map insertion order" as a byte-identical
+ * hazard for every later windowing step.
+ */
+const byGraphId = (a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+const orderedNodes = (graph, sorted) => sorted ? [...graph.iterNodes()].sort(byGraphId) : graph.iterNodes();
+const orderedRelationships = (graph, sorted) => sorted ? [...graph.iterRelationships()].sort(byGraphId) : graph.iterRelationships();
 /** Flush buffered rows to disk every N rows */
 const FLUSH_EVERY = 500;
 // ============================================================================
@@ -185,6 +200,9 @@ class BufferedCSVWriter {
  * File contents are lazy-read from disk with a generous LRU cache.
  */
 export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
+    // Deterministic (id-sorted) node/relationship row order when enabled;
+    // default off = today's graph-insertion order (byte-identical).
+    const sortOutput = parseTruthyEnv(process.env.GITNEXUS_SORT_GRAPH_OUTPUT);
     // Remove stale CSVs from previous crashed runs, then recreate
     try {
         await fs.rm(csvDir, { recursive: true, force: true });
@@ -253,7 +271,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
     // A single Set covering every label prevents PK violations on COPY.
     const seenNodeIds = new Set();
     // --- SINGLE PASS over all nodes ---
-    for (const node of graph.iterNodes()) {
+    for (const node of orderedNodes(graph, sortOutput)) {
         if (seenNodeIds.has(node.id))
             continue;
         seenNodeIds.add(node.id);
@@ -421,7 +439,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
     // --- Stream relationship CSV ---
     const relCsvPath = path.join(csvDir, 'relations.csv');
     const relWriter = new BufferedCSVWriter(relCsvPath, 'from,to,type,confidence,reason,step');
-    for (const rel of graph.iterRelationships()) {
+    for (const rel of orderedRelationships(graph, sortOutput)) {
         await relWriter.addRow([
             escapeCSVField(rel.sourceId),
             escapeCSVField(rel.targetId),