@lincoln504/pi-research 1.1.2 → 1.2.0

package/src/infrastructure/browser/browser-task-scheduler.ts

@@ -24,6 +24,52 @@ import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowser
 import { raceWithDeadline } from '../../utils/safe-unref.ts';
 import { PriorityTaskQueue } from './priority-task-queue.ts';
 
+/**
+ * Grace added to the worker's own task budget before the dead-worker backstop
+ * fires, so a live worker's more informative timeout always answers first.
+ */
+export const WORKER_DEATH_BACKSTOP_GRACE_MS = 5000;
+
+/**
+ * Execute a pool task with a dead-worker backstop.
+ *
+ * poolifier (measured on 5.3.2) never settles an in-flight execute() whose worker
+ * dies — not when the worker exits, not when a replacement is respawned, not even
+ * when the pool is destroyed. The worker enforces taskTimeoutMs on itself, but only
+ * while it is alive; an OOM-killed worker leaves the promise pending forever, and
+ * with it the PriorityTaskQueue concurrency slot the dispatched closure holds. Each
+ * such death would permanently shrink the queue until the leader serves nothing but
+ * timeouts.
+ *
+ * The backstop timer is created inside the dispatched closure, so it shares no
+ * lifetime with the caller's outer race — whose `finally` clearing a shared timer
+ * is exactly how a QUEUED task once lost its guard. Armed at dispatch, it fires
+ * only when nothing is left to answer; its rejection frees the queue slot (the
+ * caller has already been answered by the outer race).
+ */
+export async function executePoolTaskWithDeathBackstop<T>(
+    execute: () => Promise<T>,
+    taskTimeoutMs: number,
+    label: string,
+): Promise<T> {
+    const backstopMs = taskTimeoutMs + WORKER_DEATH_BACKSTOP_GRACE_MS;
+    let backstopId: NodeJS.Timeout | undefined;
+    try {
+        return await Promise.race([
+            execute(),
+            new Promise<never>((_, reject) => {
+                backstopId = setTimeout(
+                    () => reject(new Error(`${label} task unanswered after ${backstopMs}ms — worker likely died mid-task`)),
+                    backstopMs,
+                );
+                if (backstopId.unref) backstopId.unref();
+            }),
+        ]);
+    } finally {
+        if (backstopId) clearTimeout(backstopId);
+    }
+}
+
 /**
  * Browser task scheduler - manages the worker pool and executes tasks.
  * Only the leader process has an instance of this scheduler.
@@ -192,17 +238,29 @@ export class BrowserTaskScheduler implements IScheduler {
 
         logger.debug(`[BrowserTaskScheduler] Executing search: "${query}" (Timeout: ${timeoutMs}ms)`);
         let result: any;
+        // Settling the outer race must also release the task's queue claim: a task
+        // still QUEUED when the caller is answered is cancelled before it consumes
+        // a slot, and a RUNNING one frees its slot instead of keeping it (plus a
+        // fresh worker budget) for a result nobody will consume. The abort-while-
+        // queued/running semantics are the queue's own, pinned by its tests.
+        const settled = new AbortController();
+        const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
             const queue = this.getPriorityQueue(config);
             // Race the enqueue call against the timeoutPromise.
             // This ensures that even if the queue is saturated, we won't hang forever.
             result = await Promise.race([
                 queue.enqueue('search', async () => {
-                    return await Promise.race([
-                        pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutPromise
-                    ]);
-                }, signal),
+                    // The shared timeoutPromise is deliberately NOT raced in here (its timer
+                    // dies with the outer race); the backstop below carries its own timer, so
+                    // a QUEUED task can never be dispatched against a disarmed guard, and a
+                    // worker dying mid-task cannot pin the queue slot forever.
+                    return await executePoolTaskWithDeathBackstop(
+                        () => pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                        timeoutMs,
+                        'Search',
+                    );
+                }, queueSignal),
                 timeoutPromise
             ]);
             logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${Date.now() - startTime}ms`);
@@ -226,6 +284,7 @@ export class BrowserTaskScheduler implements IScheduler {
             throw error;
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
+            settled.abort();
         }
 
         const duration = Date.now() - startTime;
@@ -265,15 +324,20 @@ export class BrowserTaskScheduler implements IScheduler {
         });
 
         let result: any;
+        // See runSearch: outer settle releases the queue claim; the backstop bounds
+        // a dead worker's never-settling execute().
+        const settled = new AbortController();
+        const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
             const queue = this.getPriorityQueue(config);
             result = await Promise.race([
                 queue.enqueue('scrape', async () => {
-                    return await Promise.race([
-                        pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutPromise
-                    ]);
-                }, signal),
+                    return await executePoolTaskWithDeathBackstop(
+                        () => pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                        timeoutMs,
+                        'Scrape',
+                    );
+                }, queueSignal),
                 timeoutPromise
             ]);
         } catch (error) {
@@ -292,6 +356,7 @@ export class BrowserTaskScheduler implements IScheduler {
             throw error;
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
+            settled.abort();
         }
 
         const duration = Date.now() - startTime;
@@ -324,17 +389,20 @@ export class BrowserTaskScheduler implements IScheduler {
         });
 
         let result: { success: boolean; error?: string };
+        // See runSearch: outer settle releases the queue claim; the backstop bounds
+        // a dead worker's never-settling execute().
+        const settled = new AbortController();
+        const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
             const queue = this.getPriorityQueue(config);
             result = await Promise.race([
                 queue.enqueue('healthcheck', async () => {
-                    const execPromise = pool.execute({ type: 'healthcheck', queuedAt: startTime, taskTimeoutMs: timeoutMs });
-                    execPromise.catch((err: Error) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
-                    return await Promise.race([
-                        execPromise,
-                        timeoutPromise
-                    ]);
-                }, signal),
+                    return await executePoolTaskWithDeathBackstop(() => {
+                        const execPromise = pool.execute({ type: 'healthcheck', queuedAt: startTime, taskTimeoutMs: timeoutMs });
+                        execPromise.catch((err: Error) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
+                        return execPromise;
+                    }, timeoutMs, 'Healthcheck');
+                }, queueSignal),
                 timeoutPromise
             ]) as { success: boolean; error?: string };
         } catch (error) {
@@ -353,6 +421,7 @@ export class BrowserTaskScheduler implements IScheduler {
             throw error;
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
+            settled.abort();
         }
 
         const duration = Date.now() - startTime;
@@ -411,12 +480,21 @@ export class BrowserTaskScheduler implements IScheduler {
         // reference would be moot. Guarding here keeps shutdown from logging a
         // spurious "Cannot get service 'scheduler' during container disposal".
         if (!this.container.isDisposing) {
-            const schedulerService = await getService<ISchedulerInternals>(ServiceNames.SCHEDULER, undefined, this.container);
-            const currentScheduler = schedulerService.getSchedulerInstance();
-            if (currentScheduler && 'schedulerId' in currentScheduler && currentScheduler.schedulerId === this.schedulerId) {
-                schedulerService.setSchedulerInstance(null);
-                schedulerService.setSchedulerVersion(null);
-                schedulerService.setSchedulerInitializationPromise(null);
+            try {
+                const schedulerService = await getService<ISchedulerInternals>(ServiceNames.SCHEDULER, undefined, this.container);
+                const currentScheduler = schedulerService.getSchedulerInstance();
+                if (currentScheduler && 'schedulerId' in currentScheduler && currentScheduler.schedulerId === this.schedulerId) {
+                    schedulerService.setSchedulerInstance(null);
+                    schedulerService.setSchedulerVersion(null);
+                    schedulerService.setSchedulerInitializationPromise(null);
+                }
+            } catch (err) {
+                // Disposal can begin between the isDisposing check above and the
+                // getService call. isShuttingDown is already latched true, so a
+                // throw here would abort shutdown permanently — drain, state-clear,
+                // server-stop and pool teardown would all be skipped. The reference
+                // being unclearable is the same moot case the guard above covers.
+                logger.debug('[Scheduler] Could not clear scheduler service reference during shutdown:', err);
             }
         }
 

package/src/infrastructure/browser/cleanup-utils.ts

@@ -9,8 +9,9 @@ const DEFAULT_STALE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
  * Firefox/Camoufox profile lock files, cross-platform. A live browser holds:
  *  - Unix:   `lock` — a SYMLINK whose target encodes the owner: "<ip>:+<pid>" or "+<pid>",
  *            plus `.parentlock` (an fcntl-locked empty file).
- *  - Windows: `parent.lock` — a file kept OPEN by the live process (so the OS refuses to
- *            delete it, which is itself a safety net against reclaiming an in-use profile).
+ *  - Windows: `parent.lock` — a file kept OPEN by the live process. That is a liveness
+ *            SIGNAL (see isWindowsParentLockHeld), not a safety net: a recursive delete
+ *            has already removed the rest of the profile before it reaches this file.
  * The previous code checked for `.lock`, which Firefox/Camoufox never creates, so its
  * lock-awareness was inert. These are the real names.
  */
@@ -20,6 +21,37 @@ const LOCK_FILE_NAMES = ['lock', '.parentlock', 'parent.lock'];
  *  so a just-starting concurrent run is not reclaimed out from under itself. */
 const LOCK_FRESHNESS_MS = 2 * 60 * 1000;
 
+/**
+ * Windows: is `parent.lock` still held open by a live browser?
+ *
+ * Firefox opens this file with no sharing, so while the browser runs ANY other
+ * open of it fails with a sharing violation. That makes the probe positive
+ * evidence of a live owner — the same standing as the Unix `lock` symlink's PID.
+ *
+ * Recency cannot substitute for it here: `parent.lock`'s mtime is stamped once at
+ * browser startup and never refreshed, so every browser alive longer than
+ * LOCK_FRESHNESS_MS read as dead — the normal case for a minutes-long research
+ * run. Nor does "the OS refuses to delete a held-open file" save the profile:
+ * `fs.rm({recursive:true})` unlinks entries one at a time, so prefs, cookies and
+ * places are already gone by the time it reaches the locked file and fails.
+ *
+ * Returns null when the answer is not knowable, which callers treat as live —
+ * sparing a leaked profile costs disk, deleting a live one corrupts a running run.
+ */
+async function isWindowsParentLockHeld(lockPath: string): Promise<boolean | null> {
+  let handle: Awaited<ReturnType<typeof fs.open>> | undefined;
+  try {
+    handle = await fs.open(lockPath, 'r');
+  } catch (e) {
+    const code = (e as NodeJS.ErrnoException)?.code;
+    if (code === 'ENOENT') return false; // vanished — no live owner
+    if (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES') return true; // held open
+    return null; // unexpected — don't guess
+  }
+  await handle.close().catch(() => {});
+  return false;
+}
+
 /** Parse the owner PID out of a Firefox `lock` symlink target ("<ip>:+<pid>" / "+<pid>"). */
 function parsePidFromLockTarget(target: string): number | null {
   const m = /\+(\d+)\s*$/.exec(target);
@@ -62,9 +94,15 @@ async function isProfileLive(profileDir: string): Promise<boolean> {
         // unreadable symlink — fall back to freshness below
       }
     }
-    // `.parentlock` / `parent.lock` (no PID encoded), or an unreadable `lock`: use recency
-    // as the liveness proxy. On Windows the file is also held open, so an attempted delete
-    // of a genuinely-live profile fails harmlessly regardless of this heuristic.
+    // Windows has no `lock` symlink, so `parent.lock` is the only signal there. Probe
+    // whether it is still held open — authoritative, unlike recency, which reads every
+    // browser older than LOCK_FRESHNESS_MS as dead because the mtime is never refreshed.
+    if (process.platform === 'win32' && name === 'parent.lock') {
+      const held = await isWindowsParentLockHeld(lockPath);
+      if (held !== null) return held;
+      return true; // indeterminate — spare the profile
+    }
+    // `.parentlock` (no PID encoded) or an unreadable `lock`: use recency as the proxy.
     return Date.now() - lst.mtimeMs < LOCK_FRESHNESS_MS;
   }
   return false;
@@ -127,7 +165,7 @@ export async function cleanupStaleProfiles(
             logger.debug(`[Cleanup] Skipping profile ${entry} — owned by a live browser`);
             continue;
           }
-          await fs.rm(fullPath, { recursive: true, force: true });
+          await fs.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
           removed++;
           continue;
         }
@@ -135,7 +173,7 @@ export async function cleanupStaleProfiles(
         // Unrecognized entry (only reachable with an explicit baseDir): be conservative —
         // only reclaim if both old AND not live, so we never delete something unknown promptly.
         if (now - stats.mtimeMs > staleMs && !(await isProfileLive(fullPath))) {
-          await fs.rm(fullPath, { recursive: true, force: true });
+          await fs.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
           removed++;
         }
       } catch (_e) {

package/src/infrastructure/browser/client-agent.ts

@@ -10,14 +10,15 @@ import * as http from 'node:http';
 /**
  * Global HTTP Agent for high-concurrency client requests
  *
- * IMPORTANT: The socket timeout must be longer than the BrowserClient request timeout (120s)
- * to prevent premature socket closure when the browser is slow or the queue is deep.
+ * IMPORTANT: The socket timeout must be longer than the BrowserClient request timeout (60s,
+ * REQUEST_TIMEOUT_MS in browser-client.ts) to prevent premature socket closure when the
+ * browser is slow or the queue is deep.
  */
 const clientAgent = new http.Agent({
     keepAlive: true,
     maxSockets: 100, // Allow up to 100 concurrent requests to the leader
     maxFreeSockets: 10,
-    // Set timeout to 180s (3x the client timeout) to handle slow browser responses
+    // 180s (3x the client timeout) to handle slow browser responses
     // and prevent "socket hang up" errors during peak load
     timeout: 180000
 });

package/src/infrastructure/browser/priority-task-queue.ts

@@ -149,11 +149,18 @@ export class PriorityTaskQueue {
         // on its own; we swallow its late settlement and free the slot now so the
         // queue can dispatch the next task immediately.
         let onAbort: (() => void) | undefined;
-        const fnPromise = task.fn();
-        // Prevent an unhandled rejection if abort wins the race and fn() rejects later.
-        fnPromise.catch(() => { /* handled via the race below or abandoned on abort */ });
 
         try {
+            // task.fn() is invoked INSIDE the try: a non-async fn that throws
+            // synchronously would otherwise escape before the finally that decrements
+            // activeCount, permanently inflating the active count (starving the queue
+            // by one slot each time) and leaving the caller's enqueue() promise
+            // unsettled. Every fn today is async, so this is a latent shape defect
+            // rather than a live one — but the correct placement costs nothing.
+            const fnPromise = task.fn();
+            // Prevent an unhandled rejection if abort wins the race and fn() rejects later.
+            fnPromise.catch(() => { /* handled via the race below or abandoned on abort */ });
+
             let result: any;
             if (task.signal) {
                 const abortPromise = new Promise<never>((_, rej) => {

package/src/infrastructure/browser/scheduler-factory.ts

@@ -282,6 +282,25 @@ export async function getScheduler(config?: Config, container: ServiceContainer
             let wonElection = false;
             let winnerPort = port;
             let winnerAuthSecret: string | undefined;
+            // Read our start-time BEFORE taking the state lock (the lookup can shell
+            // out on macOS/Windows; getCurrentProcessStartTime caches, so this is a
+            // single cheap read). Recording it alongside the PID is what makes
+            // StateManager.isPidAlive's PID-reuse guard work: that guard passes
+            // `state.browserServer.startTime` as expectedStartTime, so an election
+            // that writes only the PID leaves it permanently undefined and the guard
+            // degrades to a bare signal-0 check — which a recycled PID passes,
+            // making every follower adopt an unrelated process as the browser leader.
+            // Null on a platform where it cannot be read → the entry omits it and
+            // liveness falls back to that bare check, same as pre-existing entries.
+            const myStartTime = await (async () => {
+                try {
+                    const processLifecycle = await getService<IProcessLifecycle>(ServiceNames.PROCESS_LIFECYCLE, undefined, container);
+                    return await processLifecycle.getCurrentProcessStartTime();
+                } catch (err) {
+                    logger.debug('[Scheduler] Could not read own process start time (PID-reuse guard degrades to signal-0):', err);
+                    return null;
+                }
+            })();
             try {
                 await stateManager.updateState(async (state) => {
                     if (state.browserServer) {
@@ -294,7 +313,13 @@ export async function getScheduler(config?: Config, container: ServiceContainer
                         }
                     }
                     const secret = getBrowserServerAuthSecret();
-                    state.browserServer = { port, pid: process.pid, schedulerId, authSecret: secret };
+                    state.browserServer = {
+                        port,
+                        pid: process.pid,
+                        schedulerId,
+                        authSecret: secret,
+                        ...(myStartTime !== null ? { startTime: myStartTime } : {}),
+                    };
                     state.schedulerVersion = schedulerVersion;
                     wonElection = true;
                     return state;

package/src/infrastructure/browser/task-execution-service.ts

@@ -17,6 +17,9 @@ import { getServiceContainer, getService } from '../../core/service-registry.ts'
 import type { ServiceContainer } from '../../core/service-registry.ts';
 import { ServiceNames } from '../../core/service-interfaces.ts';
 import type { ISchedulerFactory, IScheduler } from '../../core/scheduler-factory.ts';
+import type { ISchedulerInternals } from '../../core/interfaces/scheduler-interfaces.ts';
+import type { IStateManager } from '../../core/interfaces/state-manager-interfaces.ts';
+import { BrowserClient } from './browser-client.ts';
 
 // Cooldown to prevent cascading scheduler restarts (thundering herd)
 let lastRestartTime = 0;
@@ -62,6 +65,92 @@ async function recoverFromLeaderHandover(container: ServiceContainer): Promise<v
     );
 }
 
+/**
+ * Is the leader currently registered in state still a live process — and is it
+ * the one our cached handle actually targets?
+ *
+ * Used only to decide whether the restart cooldown below may be honoured.
+ * Anything that prevents a confident "yes" (no registered leader, an unreadable
+ * state file, a service-resolution failure, a cached client aimed at a different
+ * port than the registered leader's) answers `false`, because the cost of an
+ * unnecessary re-resolve is one cheap round-trip while the cost of skipping a
+ * needed one is a task failure against a dead port.
+ */
+async function isRegisteredLeaderAliveUncached(container: ServiceContainer): Promise<boolean> {
+    try {
+        const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, undefined, container);
+        const info = await stateManager.getBrowserServer();
+        if (!info) return false;
+        // "Leader alive" is not enough: the cooldown protects the CACHED handle, so
+        // that handle must also target the registered leader. If our leader B died
+        // and another process already elected C, C is alive — but our cached client
+        // still aims at B's dead port, and honouring the cooldown would leave every
+        // retry failing against the corpse until the window expired.
+        const schedulerService = await getService<ISchedulerInternals>(ServiceNames.SCHEDULER, undefined, container);
+        const cached = schedulerService.getSchedulerInstance();
+        if (cached instanceof BrowserClient && cached.targetPort !== info.port) return false;
+        return await stateManager.isPidAlive(info.pid, info.schedulerId);
+    } catch {
+        return false;
+    }
+}
+
+/**
+ * Coalesce concurrent liveness probes.
+ *
+ * This runs on the failure path, where a whole query burst fails at once: a
+ * 20-query search losing its leader calls it up to 19 times within milliseconds
+ * (the first caller restarts and the rest land inside the cooldown). The probe is
+ * cheap on Linux — a `/proc/<pid>/stat` read — but on Windows the PID-reuse guard
+ * resolves the process start time by spawning `powershell -NoProfile`, so an
+ * uncoalesced burst would fan out into ~19 concurrent PowerShell processes on the
+ * slowest platform we support. The window is deliberately far shorter than
+ * RESTART_COOLDOWN_MS: it exists to collapse one burst, not to cache a verdict.
+ */
+const LEADER_LIVENESS_COALESCE_MS = 1000;
+let leaderLivenessProbe: { startedAt: number; result: Promise<boolean> } | null = null;
+
+async function isRegisteredLeaderAlive(container: ServiceContainer): Promise<boolean> {
+    const now = Date.now();
+    if (leaderLivenessProbe && now - leaderLivenessProbe.startedAt < LEADER_LIVENESS_COALESCE_MS) {
+        return leaderLivenessProbe.result;
+    }
+    const result = isRegisteredLeaderAliveUncached(container);
+    leaderLivenessProbe = { startedAt: now, result };
+    return result;
+}
+
+/** Test seam: drop the coalesced liveness probe. */
+export function _resetLeaderLivenessProbeForTests(): void {
+    leaderLivenessProbe = null;
+}
+
+/**
+ * Drop the scheduler handle and wait for the pool to settle, subject to a
+ * cooldown that stops N concurrent task failures from each restarting.
+ *
+ * The cooldown is only safe while the registered leader is still alive: then the
+ * cached handle points at a working process and skipping the restart costs
+ * nothing. It is NOT safe when the leader is gone. A hard leader kill (SIGKILL,
+ * OOM) surfaces to clients as ECONNREFUSED, which `isTransientSocketError`
+ * matches but `isPoolShutdownError` does not — so it lands here rather than on
+ * the handover path, and a bare wait leaves the cached client aimed at a corpse.
+ * The retry then re-resolves to the same dead port and fails identically: the
+ * exact stale-handle failure the leader-handover work fixed for the *graceful*
+ * shutdown case. So the cooldown yields whenever the leader is not confirmed alive.
+ */
+async function restartSchedulerWithHerdGuard(container: ServiceContainer): Promise<void> {
+    const now = Date.now();
+    if (now - lastRestartTime > RESTART_COOLDOWN_MS || !(await isRegisteredLeaderAlive(container))) {
+        lastRestartTime = now;
+        logger.warn('[BrowserManager] Forcing scheduler restart and retrying...');
+        await forceSchedulerRestart(false, container);
+    } else {
+        logger.warn('[BrowserManager] Scheduler restart triggered recently and the registered leader is still alive — waiting for pool idle instead...');
+    }
+    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
+}
+
 /**
  * Get the scheduler from the factory, optionally using a specific container.
  */
@@ -163,16 +252,7 @@ export async function runBrowserTask<T>(
                 // PID+start-time liveness probe confirms the registered leader is
                 // actually gone; it only skips clearing when that probe finds the
                 // leader alive, which is exactly the case we must not disturb.
-                const now = Date.now();
-                if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
-                    lastRestartTime = now;
-                    logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
-                    await forceSchedulerRestart(false, container);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                } else {
-                    logger.warn(`[BrowserManager] Scheduler restart recently triggered, waiting for pool idle...`);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                }
+                await restartSchedulerWithHerdGuard(container);
             }
 
             // Retry with jitter (100-500ms) to prevent thundering herd
@@ -211,15 +291,7 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
                 // See runBrowserTask: don't force-clear remote leader state on a bare
                 // transient socket error — let the liveness probe inside
                 // forceSchedulerRestart decide.
-                const now = Date.now();
-                if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
-                    lastRestartTime = now;
-                    logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
-                    await forceSchedulerRestart(false, container);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                } else {
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                }
+                await restartSchedulerWithHerdGuard(container);
             }
 
             const jitter = 100 + Math.floor(Math.random() * 400);
@@ -278,15 +350,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
                 // See runBrowserTask: don't force-clear remote leader state on a bare
                 // transient socket error — let the liveness probe inside
                 // forceSchedulerRestart decide.
-                const now = Date.now();
-                if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
-                    lastRestartTime = now;
-                    logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
-                    await forceSchedulerRestart(false, container);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                } else {
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                }
+                await restartSchedulerWithHerdGuard(container);
             }
 
             const jitter = 100 + Math.floor(Math.random() * 400);

package/src/infrastructure/browser/thread-worker-lifecycle.ts

@@ -206,14 +206,60 @@ export function setupOrphanProtection(): void {
     return;
   }
 
-  if (!process.ppid) {
+  // Capture the parent PID ONCE, at setup. `process.ppid` is re-read from the OS
+  // on every access, so on POSIX it changes the moment we are orphaned — and that
+  // change, not a liveness probe, is the reliable signal (see checkOrphan).
+  const originalPpid = process.ppid;
+  if (!originalPpid) {
     return;
   }
 
+  /** Tear down the browser and exit. Shared by both orphan signals below. */
+  const exitAsOrphan = async () => {
+    // Await cleanup to prevent browser/context leaks — but bounded. The browser is
+    // the plausible thing to be wedged after the parent died abruptly, and an
+    // unbounded browser.close() hang here would keep the orphan (and its browser)
+    // alive forever: the precise leak this guard exists to stop. The exit deadline
+    // wins over a perfect teardown.
+    if (cleanupBrowser) {
+      await Promise.race([
+        cleanupBrowser().catch(err => {
+          logToDebugFile('WARN', `[Worker-${workerId}] Browser cleanup failed during orphan exit:`, err);
+        }),
+        new Promise<void>(resolve => {
+          const t = setTimeout(resolve, 10_000);
+          if (t.unref) t.unref();
+        }),
+      ]);
+    }
+    // Clear the orphan check timer
+    cleanupOrphanProtection();
+    process.env['PI_PROCESS_EXITING'] = '1';
+    process.exit(1);
+  };
+
   const checkOrphan = async () => {
     try {
-      // signal 0 checks if the process is alive
-      process.kill(process.ppid, 0);
+      // Re-parenting is the authoritative orphan signal on POSIX. When the parent
+      // dies the kernel immediately re-parents us — to PID 1, or to the nearest
+      // subreaper (a user systemd instance, a container init). Probing the CURRENT
+      // ppid cannot see that: PID 1 always exists, and signalling it from an
+      // unprivileged process raises EPERM, which the handler below correctly reads
+      // as "alive but not ours to signal". Net effect on Linux was that this guard
+      // never fired at all and orphaned workers kept their browsers running.
+      // Comparing against the PID captured at setup detects it directly.
+      // Windows does not re-parent, so the liveness probe below remains the
+      // mechanism there and this check simply never trips.
+      if (process.ppid !== originalPpid) {
+        logToDebugFile('WARN', `[Worker-${workerId}] Re-parented (${originalPpid} -> ${process.ppid}): parent died, shutting down...`);
+        await exitAsOrphan();
+        return;
+      }
+
+      // signal 0 checks if the process is alive. Probe the ORIGINAL ppid, never
+      // `process.ppid` — after a re-parent the latter points at init, not at the
+      // process whose liveness we actually care about.
+      process.kill(originalPpid, 0);
 
       // Schedule next check
       orphanCheckTimer = setTimeout(checkOrphan, 10000);
@@ -231,16 +277,7 @@ export function setupOrphanProtection(): void {
       }
       // If error is thrown, the parent process is likely dead or unreachable
       logToDebugFile('WARN', `[Worker-${workerId}] Parent process died or unreachable (orphaned), shutting down...`);
-      // FIX: Await cleanup to prevent browser/context leaks
-      if (cleanupBrowser) {
-        await cleanupBrowser().catch(err => {
-          logToDebugFile('WARN', `[Worker-${workerId}] Browser cleanup failed during orphan exit:`, err);
-        });
-      }
-      // Clear the orphan check timer
-      cleanupOrphanProtection();
-      process.env['PI_PROCESS_EXITING'] = '1';
-      process.exit(1);
+      await exitAsOrphan();
     }
   };