@lincoln504/pi-research 1.3.12 → 1.4.3

package/src/infrastructure/browser/browser-task-scheduler.ts

@@ -19,10 +19,11 @@ import type { IStateManager } from '../../core/interfaces/state-manager-interfac
 import { BrowserServer, getBrowserServerAuthSecret } from './browser-server.ts';
 import { isPoolShutdownError } from './browser-error-utils.ts';
 import type { WorkerPoolManager } from './worker-pool-manager.ts';
-import type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
+import type { IScheduler, TaskDispatchListener } from '../../core/interfaces/scheduler-interfaces.ts';
 import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowserProcesses } from './browser-cleanup.ts';
 import { raceWithDeadline } from '../../utils/safe-unref.ts';
 import { PriorityTaskQueue } from './priority-task-queue.ts';
+import { getHealthCheckBudgetMs, COLD_START_ALLOWANCE_MS } from './config.ts';
 
 /**
  * Grace added to the worker's own task budget before the dead-worker backstop
@@ -74,6 +75,47 @@ export async function executePoolTaskWithDeathBackstop<T>(
  * Browser task scheduler - manages the worker pool and executes tasks.
  * Only the leader process has an instance of this scheduler.
  */
+/**
+ * How long a task actually RAN, given when it was dispatched and when it was
+ * submitted.
+ *
+ * Duration used to be `now - submittedAt`, which folds queue wait into the work.
+ * That is the number `Search completed ... in Nms` prints and the number
+ * `browser_search_duration_ms` records, so it is what every latency investigation
+ * reads — and during a saturated burst it reported a 51s median against a true
+ * 5.8s execution median. A whole evening went into explaining a slowdown that had
+ * not happened. Moving the deadline off queue wait while leaving the measurement
+ * on it would have preserved exactly that misdiagnosis.
+ *
+ * A task that never reached a worker has no execution time; report its full
+ * elapsed span rather than zero, so a timeout is not silently recorded as instant.
+ */
+export function executionMs(dispatchedAt: number, submittedAt: number, now: number = Date.now()): number {
+    return dispatchedAt > 0 ? now - dispatchedAt : now - submittedAt;
+}
+
+/** How long a task sat in the queue before a worker took it. Zero if never dispatched. */
+export function queueWaitMs(dispatchedAt: number, submittedAt: number): number {
+    return dispatchedAt > 0 ? dispatchedAt - submittedAt : 0;
+}
+
+/**
+ * Record a task's queue wait, whether or not it was ever dispatched.
+ *
+ * Split by outcome because the two are different quantities and must not be summed
+ * into one distribution: a dispatched task's wait ENDED, while an abandoned task's
+ * wait was still running when the caller gave up, so the latter is a lower bound.
+ * Folding abandonments in as `queueWaitMs`'s zero would pull the distribution
+ * DOWNWARD in exactly the bursts where it should spike.
+ */
+export function observeQueueWait(metric: string, dispatchedAt: number, submittedAt: number): void {
+    metrics.observe(
+        metric,
+        dispatchedAt > 0 ? dispatchedAt - submittedAt : Date.now() - submittedAt,
+        { dispatched: dispatchedAt > 0 ? 'yes' : 'no' },
+    );
+}
+
 export class BrowserTaskScheduler implements IScheduler {
     private workerPoolManager: WorkerPoolManager | null = null;
     private server: BrowserServer | null = null;
@@ -169,6 +211,9 @@ export class BrowserTaskScheduler implements IScheduler {
                 // Decay (not reset) so a slow worker crash-loop can still accumulate to
                 // the auto-recovery threshold instead of being zeroed on every tick.
                 poolManager.decayConsecutiveErrors();
+                // Sampled on the leadership tick rather than per task: this is a
+                // distribution property, not a per-task one, and it should read zero.
+                poolManager.recordDispatchHealth();
             } catch (err) {
                 logger.warn('[Scheduler] Leadership check error:', err);
             } finally {
@@ -218,7 +263,7 @@ export class BrowserTaskScheduler implements IScheduler {
         return this.server.start();
     }
 
-    async runSearch(query: string, config?: Config, signal?: AbortSignal): Promise<SearchResult[]> {
+    async runSearch(query: string, config?: Config, signal?: AbortSignal, onTaskEvent?: TaskDispatchListener): Promise<SearchResult[]> {
         // Reject tasks once this scheduler has begun teardown. shutdown() nulls priorityQueue and
         // clears the scheduler reference (a future run gets a fresh scheduler), so getPriorityQueue()
         // here would otherwise build a FRESH, non-shutdown queue on this dead instance and dispatch
@@ -228,19 +273,48 @@ export class BrowserTaskScheduler implements IScheduler {
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
         const startTime = Date.now();
 
-        // Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + a queue-wait/overhead
-        // margin (BROWSER_TASK_TIMEOUT_MS). Deriving from SEARCH_TIMEOUT_MS keeps the task
-        // ceiling coherent with the worker's own nav timeout — using a flat base here would
-        // kill a search before it had used its full nav budget.
+        // Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + an overhead margin
+        // (BROWSER_TASK_TIMEOUT_MS) + a cold-start allowance (COLD_START_ALLOWANCE_MS).
+        // Deriving from SEARCH_TIMEOUT_MS keeps the task ceiling coherent with the
+        // worker's own nav timeout — using a flat base here would kill a search before
+        // it had used its full nav budget. The cold-start term matters because eager
+        // browser warmup is deliberately disabled: the first search dispatched to a
+        // freshly created or just-reset worker pays a real Camoufox launch + context
+        // creation inline, before any navigation even starts — without this term that
+        // legitimate cold start reads as a dead/wedged worker (see COLD_START_ALLOWANCE_MS).
         const cfg = config || getConfig();
-        const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
-
+        const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS;
+
+        // The deadline measures EXECUTION, not queue wait: the timer is armed by
+        // `startDeadline()` from inside the enqueued callback, i.e. once the queue has
+        // actually dispatched this task to a worker.
+        //
+        // It used to be armed here, before enqueuing, so a saturated queue burned a task's
+        // entire budget waiting for a slot and it aborted having done no work at all. That
+        // made the number of tasks a burst could complete `capacity x (budget / latency)`
+        // rather than the number submitted — measured on a live run, 78 of 100 searches
+        // aborted at the deadline without ever running. A queue is supposed to absorb
+        // volume by making work wait, not by killing whatever it could not reach in time.
+        // Waiting is now bounded by the caller's signal and the run-level timeouts; a task
+        // that does get dispatched always gets its full budget.
+        // `dispatchedAt` is the same instant, kept for MEASUREMENT rather than for the
+        // deadline. Reporting `Date.now() - startTime` as the search's duration charged
+        // queue wait to the work, and that number is what every latency analysis reads:
+        // on the night this was diagnosed it reported a 51s median against a true 5.8s
+        // execution median, and invented an eightfold latency regression that never
+        // happened. Moving the deadline off queue wait without moving the measurement
+        // would have left the misdiagnosis in place while claiming to have fixed it.
         let timeoutId: NodeJS.Timeout | undefined;
+        let dispatchedAt = 0;
+        let startDeadline: () => void = () => {};
         const timeoutPromise = new Promise<never>((_, reject) => {
-            timeoutId = setTimeout(() => {
-                reject(new Error(`Search task timed out after ${timeoutMs}ms (including queue wait). query="${query}"`));
-            }, timeoutMs);
-            if (timeoutId.unref) timeoutId.unref();
+            startDeadline = () => {
+                dispatchedAt = Date.now();
+                timeoutId = setTimeout(() => {
+                    reject(new Error(`Search task timed out after ${timeoutMs}ms. query="${query}"`));
+                }, timeoutMs);
+                if (timeoutId.unref) timeoutId.unref();
+            };
         });
 
         logger.debug(`[BrowserTaskScheduler] Executing search: "${query}" (Timeout: ${timeoutMs}ms)`);
@@ -258,19 +332,24 @@ export class BrowserTaskScheduler implements IScheduler {
             // This ensures that even if the queue is saturated, we won't hang forever.
             result = await Promise.race([
                 queue.enqueue('search', async () => {
-                    // The shared timeoutPromise is deliberately NOT raced in here (its timer
-                    // dies with the outer race); the backstop below carries its own timer, so
-                    // a QUEUED task can never be dispatched against a disarmed guard, and a
-                    // worker dying mid-task cannot pin the queue slot forever.
-                    return await executePoolTaskWithDeathBackstop(
-                        () => pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutMs,
-                        'Search',
-                    );
+                    startDeadline();
+                    // Same instant, for the same reason, one layer up: whoever called us
+                    // arms their own guard here instead of at call time, and disarms it
+                    // when we settle. A listener that throws must not take the task down.
+                    try { onTaskEvent?.('dispatched'); } catch { /* caller's guard, caller's problem */ }
+                    try {
+                        return await executePoolTaskWithDeathBackstop(
+                            () => pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                            timeoutMs,
+                            'Search',
+                        );
+                    } finally {
+                        try { onTaskEvent?.('settled'); } catch { /* as above */ }
+                    }
                 }, queueSignal),
                 timeoutPromise
             ]);
-            logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${Date.now() - startTime}ms`);
+            logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${executionMs(dispatchedAt, startTime)}ms (queued ${queueWaitMs(dispatchedAt, startTime)}ms)`);
         } catch (error) {
             // A pool-shutdown/destroy error means the run is being torn down (quit/SIGTERM
             // mid-search-burst): the task never ran because dispose() destroyed the pool.
@@ -304,9 +383,16 @@ export class BrowserTaskScheduler implements IScheduler {
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
             settled.abort();
+            // Recorded here rather than on the success path below, because the tasks
+            // that wait longest are precisely the ones that do not succeed — measuring
+            // queue wait only on completed tasks censors the tail this metric exists to
+            // expose. A task that never reached a worker has no dispatch instant, so its
+            // wait is the whole elapsed time and is labelled as such; averaging it in as
+            // a zero would have understated saturation exactly when it was worst.
+            observeQueueWait('browser_search_queue_wait_ms', dispatchedAt, startTime);
         }
 
-        const duration = Date.now() - startTime;
+        const duration = executionMs(dispatchedAt, startTime);
         // In-band worker error checked FIRST: a task the worker reports as failed
         // must count only as an error. Recording success unconditionally before
         // this check double-counted every worker-reported failure as
@@ -327,23 +413,32 @@ export class BrowserTaskScheduler implements IScheduler {
         return result.results;
     }
 
-    async runScrape(url: string, config?: Config, signal?: AbortSignal): Promise<ScrapeResult> {
+    async runScrape(url: string, config?: Config, signal?: AbortSignal, onTaskEvent?: TaskDispatchListener): Promise<ScrapeResult> {
         if (this.isShuttingDown) throw new Error('Worker pool is shutting down');
         this.resetIdleTimer();
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
         const startTime = Date.now();
-        // Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + the queue-wait/overhead
-        // margin (BROWSER_TASK_TIMEOUT_MS), mirroring the search path so both stay coherent.
+        // Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + an overhead margin,
+        // mirroring the search path so both stay coherent. The mocked branch skips the
+        // cold-start allowance deliberately: PI_RESEARCH_MOCK_SCRAPE implies no real
+        // Camoufox launch is expected for scrape tasks, so there is nothing to budget for.
         const cfg = config || getConfig();
         const isMocking = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
-        const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS);
+        const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS);
 
+        // Armed on dispatch, not on enqueue — see runSearch for why.
+        // See runSearch: dispatch time is tracked for measurement, not for the deadline.
         let timeoutId: NodeJS.Timeout | undefined;
+        let dispatchedAt = 0;
+        let startDeadline: () => void = () => {};
         const timeoutPromise = new Promise<never>((_, reject) => {
-            timeoutId = setTimeout(() => {
-                reject(new Error(`Scrape task timed out after ${timeoutMs}ms (including queue wait). url="${url}"`));
-            }, timeoutMs);
-            if (timeoutId.unref) timeoutId.unref();
+            startDeadline = () => {
+                dispatchedAt = Date.now();
+                timeoutId = setTimeout(() => {
+                    reject(new Error(`Scrape task timed out after ${timeoutMs}ms. url="${url}"`));
+                }, timeoutMs);
+                if (timeoutId.unref) timeoutId.unref();
+            };
         });
 
         let result: any;
@@ -355,11 +450,18 @@ export class BrowserTaskScheduler implements IScheduler {
             const queue = this.getPriorityQueue(config);
             result = await Promise.race([
                 queue.enqueue('scrape', async () => {
-                    return await executePoolTaskWithDeathBackstop(
-                        () => pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutMs,
-                        'Scrape',
-                    );
+                    startDeadline();
+                    // See runSearch: the caller's guard is armed on dispatch and disarmed on settle.
+                    try { onTaskEvent?.('dispatched'); } catch { /* caller's guard, caller's problem */ }
+                    try {
+                        return await executePoolTaskWithDeathBackstop(
+                            () => pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                            timeoutMs,
+                            'Scrape',
+                        );
+                    } finally {
+                        try { onTaskEvent?.('settled'); } catch { /* as above */ }
+                    }
                 }, queueSignal),
                 timeoutPromise
             ]);
@@ -386,9 +488,11 @@ export class BrowserTaskScheduler implements IScheduler {
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
             settled.abort();
+            // See runSearch: recorded on every outcome, not only on success.
+            observeQueueWait('browser_scrape_queue_wait_ms', dispatchedAt, startTime);
         }
 
-        const duration = Date.now() - startTime;
+        const duration = executionMs(dispatchedAt, startTime);
         // In-band error first — success-side recording must not precede it (see runSearch).
         if (result.error) {
             metrics.increment('browser_scrape_requests_total', 1, { status: 'error' });
@@ -405,17 +509,103 @@ export class BrowserTaskScheduler implements IScheduler {
         return result;
     }
 
-    async runHealthCheck(config?: Config, signal?: AbortSignal): Promise<{ success: boolean }> {
+    async runHealthCheck(config?: Config, signal?: AbortSignal, budgetOverrideMs?: number): Promise<{ success: boolean }> {
         if (this.isShuttingDown) throw new Error('Worker pool is shutting down');
         this.resetIdleTimer();
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
         const startTime = Date.now();
         const isMocking = process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' || process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
-        const timeoutMs = (45000 + 60000) / (isMocking ? 4 : 1);
+        // Derived, not hardcoded — see getHealthCheckBudgetMs. The wait-phase verdict
+        // below rests on this budget out-waiting any task that can hold a slot, and a
+        // fixed 105s made that true only for the default timeouts. `budgetOverrideMs`
+        // exists so the verdict is reachable from a test at all: at 105s it is not,
+        // which is exactly how a previous version of it shipped unreachable and stayed
+        // that way.
+        const timeoutMs = budgetOverrideMs ?? getHealthCheckBudgetMs(config) / (isMocking ? 4 : 1);
+
+        // A probe that never got a worker used to fail as "timed out (including queue
+        // wait)", and the registry reported BrowserRuntime CRITICALLY UNHEALTHY —
+        // during a burst in which the pool was healthily executing the run's own
+        // searches. Twice on the night this was measured.
+        //
+        // Arming the deadline on DISPATCH is the whole fix, and it is sufficient. Three
+        // attempts were made to add a second, wait-phase verdict that would excuse a
+        // saturated pool, and all three were wrong. They are recorded here because each
+        // one reads sensibly and the next reader will otherwise reach for one of them:
+        //
+        //   1. "did the queue dispatch anything recently" — unreachable. Healthcheck
+        //      tasks have strict queue priority, so if a slot frees while this probe
+        //      waits, the queue dispatches THIS probe and cancels the timer. The timer
+        //      can only fire in a window where nothing dispatched at all.
+        //   2. "are the slots occupied" — wrong in the dangerous direction. activeCount
+        //      rises before pool.execute, so six workers hung inside a browser launch
+        //      are six occupied slots, and the exact incident this was written for
+        //      would have reported HEALTHY.
+        //   3. "has a task completed successfully recently" — unreachable for the same
+        //      reason as (1). A success frees a slot, a freed slot dispatches this
+        //      probe, so at expiry the last success is necessarily older than the whole
+        //      budget. The condition it tested was false by construction.
+        //
+        // The general form of the mistake: with strict priority, a probe that waited
+        // out its entire budget without being dispatched IS the evidence of a wedge.
+        // There is no second signal to consult, and every attempt to find one either
+        // could not fire or fired for a dead pool. The numbers below are kept for the
+        // DIAGNOSTIC message — an operator needs to know whether the slots were busy
+        // and when work last succeeded — and not for the verdict.
+        //
+        // False positives are bounded by arithmetic rather than by judgement: the probe
+        // allows 105s while every task that can hold a slot is itself deadlined well
+        // under that (search 55s, scrape 25s plus backstop), so a slot frees inside the
+        // budget unless the pool has genuinely stopped.
+        //
+        // Once dispatched, the deadline measures execution alone (armed below, like
+        // runSearch/runScrape), so a slow probe still fails on its own merits.
         let timeoutId: NodeJS.Timeout | undefined;
+        let dispatched = false;
+        let hcDispatchedAt = 0;
+        let startDeadline: () => void = () => {};
+        // Captured BEFORE the timer can run. Resolving the queue inside the callback
+        // would call getPriorityQueue() on a scheduler that may have shut down in the
+        // meantime, which builds a FRESH queue on the dead instance — undoing the
+        // isShutdown latch that exists to reject enqueues racing teardown, and
+        // reporting a routine shutdown as a wedge because a new queue has dispatched
+        // nothing.
+        const healthQueue = this.getPriorityQueue(config);
+        // Rejects only. It used to be typed as resolvable because a branch here answered
+        // "healthy" without running the probe; nothing does that now, and the type says so.
         const timeoutPromise = new Promise<never>((_, reject) => {
-            timeoutId = setTimeout(() => reject(new Error(`Health check timed out after ${timeoutMs}ms (including queue wait)`)), timeoutMs);
+            const onWaitExpired = () => {
+                if (dispatched) return;
+                const lastSuccessAt = healthQueue.getLastSuccessAt();
+                const sinceSuccess = lastSuccessAt > 0 ? Date.now() - lastSuccessAt : Number.POSITIVE_INFINITY;
+                // The wording is load-bearing, not prose. isBusyPoolHealthFailure
+                // (src/healthcheck/index.ts) decides whether a failed probe aborts the
+                // run, and it decides by matching /tim(e|ed) out|timeout/ on this very
+                // string. An earlier rewrite of this message dropped the word entirely —
+                // so a probe that queued out under load stopped reading as a timeout, the
+                // tolerance gate rejected it, and a BUSY pool aborted the run at
+                // quick-research-orchestrator and research-health. The commit that did it
+                // was the one meant to stop a busy pool being reported as dead; it moved
+                // the abort one layer up. Pinned by a test that drives this exact string
+                // through that exact classifier.
+                //
+                // The rest is diagnosis: slots busy with nothing completing is a worker
+                // fault, slots free is a queue fault (the queue stopped dispatching), and
+                // the last-success age separates "stopped just now" from "never started".
+                reject(new Error(
+                    `Health check timed out after ${timeoutMs}ms without reaching a worker — ${healthQueue.getActiveCount()} of ${healthQueue.getConcurrency()} slots busy, ` +
+                    `${lastSuccessAt > 0 ? `last successful task ${sinceSuccess}ms ago` : 'no task has ever completed successfully'}.`,
+                ));
+            };
+            timeoutId = setTimeout(onWaitExpired, timeoutMs);
             if (timeoutId.unref) timeoutId.unref();
+            startDeadline = () => {
+                dispatched = true;
+                hcDispatchedAt = Date.now();
+                clearTimeout(timeoutId);
+                timeoutId = setTimeout(() => reject(new Error(`Health check timed out after ${timeoutMs}ms.`)), timeoutMs);
+                if (timeoutId.unref) timeoutId.unref();
+            };
         });
 
         let result: { success: boolean; error?: string };
@@ -424,9 +614,11 @@ export class BrowserTaskScheduler implements IScheduler {
         const settled = new AbortController();
         const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
-            const queue = this.getPriorityQueue(config);
+            // Same instance the wait-expiry message reads, so the diagnosis cannot
+            // describe a different queue from the one this probe waited on.
             result = await Promise.race([
-                queue.enqueue('healthcheck', async () => {
+                healthQueue.enqueue('healthcheck', async () => {
+                    startDeadline();
                     return await executePoolTaskWithDeathBackstop(() => {
                         const execPromise = pool.execute({ type: 'healthcheck', queuedAt: startTime, taskTimeoutMs: timeoutMs });
                         execPromise.catch((err: Error) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
@@ -459,9 +651,10 @@ export class BrowserTaskScheduler implements IScheduler {
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
             settled.abort();
+            observeQueueWait('browser_healthcheck_queue_wait_ms', hcDispatchedAt, startTime);
         }
 
-        const duration = Date.now() - startTime;
+        const duration = executionMs(hcDispatchedAt, startTime);
         // In-band error first — success-side recording must not precede it (see
         // runSearch). This path additionally owns the browser_pool_health gauge:
         // setting it to 1 before the check flickered the pool healthy on every

package/src/infrastructure/browser/config.ts

@@ -327,4 +327,82 @@ export function isBrowserAvailable(): boolean {
 function isFullMockMode(): boolean {
     return process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' &&
            process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
-}
+}
+/**
+ * How long the leader's health probe may take, end to end.
+ *
+ * This was a hardcoded 105s (45s probe + 60s queue-wait margin) and the verdict logic
+ * that reads it is built on an arithmetic claim: the probe out-waits any task that can
+ * hold a worker slot, so a probe that never reaches a worker really has found a wedge.
+ * That claim held for the DEFAULT timeouts and for no others. `SEARCH_TIMEOUT_MS` and
+ * `BROWSER_TASK_TIMEOUT_MS` are both settable to 120 000, so a search may legally hold
+ * a slot for 240s — and a user who merely raised the search timeout would see healthy
+ * pools reported as wedged, which aborts the run at the readiness gate.
+ *
+ * Deriving it instead makes the invariant hold by construction:
+ *   - the longest a single task may hold a slot, plus
+ *   - the probe's own work: the worker runs up to THREE navigation attempts (primary,
+ *     retry, neutral-endpoint fallback) at `max(10s, HEALTH_CHECK_TIMEOUT_MS)` each,
+ *     and a budget below that sum makes the fallback ladder unreachable — so raising
+ *     HEALTH_CHECK_TIMEOUT_MS used to make the health check strictly worse.
+ *
+ * Floored at the historical 105s, kept as a floor rather than a default: the
+ * derived value now exceeds it even at default timeouts once COLD_START_ALLOWANCE_MS
+ * (below) is folded into `longestSlotHold` — a cold Firefox launch was always able to
+ * hold a slot for close to that long, the 105s floor just didn't know it. The floor
+ * still matters for a hypothetical config where the derived value would otherwise land
+ * below it.
+ */
+export const HEALTHCHECK_MIN_BUDGET_MS = 45_000 + 60_000;
+
+/** Navigation attempts executeHealthCheck makes: primary, retry, fallback endpoint. */
+const HEALTHCHECK_NAV_ATTEMPTS = 3;
+
+/**
+ * Hard ceiling on a single Camoufox browser launch (see launchOnce() in
+ * thread-worker-browser.ts, the single source of truth this mirrors — CI runners
+ * (2 vCPU) can take up to ~60s to start Firefox under load, floored generously
+ * above that).
+ */
+export const BROWSER_LAUNCH_TIMEOUT_MS = 90_000;
+
+/**
+ * Hard ceiling on creating one BrowserContext against an already-launched browser
+ * (see acquireTaskContext() in thread-worker-browser.ts, the single source of
+ * truth this mirrors).
+ */
+export const CONTEXT_CREATION_TIMEOUT_MS = 30_000;
+
+/**
+ * Worst-case time a worker can spend getting a browser+context ready before it
+ * starts a task's actual work. Eager warmup is deliberately disabled (thundering-
+ * herd avoidance — see thread-worker.ts), so the first task dispatched to a
+ * freshly created or just-reset worker pays this in full, inline with the task
+ * itself. Every deadline that bounds how long a task may legitimately hold a
+ * worker slot has to include this on top of its own nav budget, or a completely
+ * normal cold start on a fresh pool reads as a dead/wedged worker — the false
+ * positive `runSearch`/`runScrape`'s task-timeout ceiling and this budget were
+ * both built to eliminate, just triggered by a different clock.
+ *
+ * Deliberately NOT part of BROWSER_TASK_TIMEOUT_MS: that value is user-tunable
+ * overhead for per-task work, and cold-start cost isn't something a user should
+ * need to account for by hand when adjusting it.
+ */
+export const COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
+
+export function getHealthCheckBudgetMs(config?: Config): number {
+    const c = (config || getConfig()) as Partial<Config>;
+    // Each field is read defensively. A missing or non-finite value would otherwise
+    // propagate a NaN into `setTimeout`, which Node coerces to 1ms — turning the health
+    // probe's deadline into an instant failure. Falling back to the schema default is
+    // the only reading that cannot silently invert the bound.
+    const ms = (value: unknown, fallback: number): number =>
+        typeof value === 'number' && Number.isFinite(value) ? value : fallback;
+    const longestSlotHold =
+        Math.max(ms(c.SEARCH_TIMEOUT_MS, 45_000), ms(c.SCRAPE_TIMEOUT_MS, 15_000))
+        + ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000)
+        + COLD_START_ALLOWANCE_MS;
+    // Mirrors the worker's own floor in executeHealthCheck.
+    const probeWork = HEALTHCHECK_NAV_ATTEMPTS * Math.max(10_000, ms(c.HEALTH_CHECK_TIMEOUT_MS, 10_000));
+    return Math.max(HEALTHCHECK_MIN_BUDGET_MS, longestSlotHold + probeWork);
+}

package/src/infrastructure/browser/priority-task-queue.ts

@@ -2,6 +2,20 @@ import { logger } from '../../logger.ts';
 
 export type TaskType = 'search' | 'scrape' | 'healthcheck';
 
+/**
+ * Did this resolved value carry an in-band worker failure?
+ *
+ * Worker tasks report failure by RESOLVING with `{ error: ... }` rather than by
+ * rejecting, so "the promise settled successfully" and "the task succeeded" are
+ * different facts about the same value.
+ */
+function isTaskFailureResult(result: unknown): boolean {
+    return typeof result === 'object'
+        && result !== null
+        && typeof (result as { error?: unknown }).error === 'string'
+        && (result as { error: string }).error.length > 0;
+}
+
 export interface QueuedTask<T> {
     type: TaskType;
     fn: () => Promise<T>;
@@ -27,6 +41,16 @@ export class PriorityTaskQueue {
     private maxTotalConcurrency: number;
     private readonly maxQueueDepth: number;
     private isShutdown = false;
+    /** When a task last completed WITHOUT error (epoch ms), or 0 if never.
+     *
+     *  Diagnostic only — it is reported in the health probe's failure message so an
+     *  operator can tell a pool that stopped a moment ago from one that never started.
+     *  It was briefly used as a health VERDICT and must not be again: a success frees a
+     *  slot, a freed slot dispatches the waiting probe (healthchecks have strict
+     *  priority), so by the time a probe times out this value is necessarily older than
+     *  its whole budget. Failures deliberately do not count either — a wedged pool still
+     *  turns its slots over as deadlines and death-backstops fire. */
+    private lastSuccessAt = 0;
 
     constructor(maxTotalConcurrency: number, maxQueueDepth = 500) {
         this.maxTotalConcurrency = maxTotalConcurrency;
@@ -90,6 +114,21 @@ export class PriorityTaskQueue {
         });
     }
 
+    /** Epoch ms of the last task that completed successfully, or 0 if none has. */
+    getLastSuccessAt(): number {
+        return this.lastSuccessAt;
+    }
+
+    /** Tasks currently executing on a worker. */
+    getActiveCount(): number {
+        return this.activeCount;
+    }
+
+    /** How many tasks may execute at once. */
+    getConcurrency(): number {
+        return this.maxTotalConcurrency;
+    }
+
     private removeFromQueue(task: QueuedTask<any>): boolean {
         const queues = [this.healthcheckQueue, this.searchQueue, this.scrapeQueue];
         for (const q of queues) {
@@ -171,6 +210,13 @@ export class PriorityTaskQueue {
             } else {
                 result = await fnPromise;
             }
+            // A worker-reported failure arrives as a RESOLVED value carrying `error` —
+            // poolifier delivers it that way, and the scheduler only turns it into a
+            // throw afterwards. Resolution alone therefore does not mean the task
+            // succeeded: a worker returning `{error:'Task timed out'}` for every task
+            // would keep this timestamp fresh through a total outage, which is the one
+            // thing it must not do.
+            if (!isTaskFailureResult(result)) this.lastSuccessAt = Date.now();
             task.resolve(result);
         } catch (err) {
             task.reject(err);
@@ -187,8 +233,16 @@ export class PriorityTaskQueue {
      * Update the maximum concurrency limit (e.g. if config changes).
      */
     updateConcurrency(maxTotalConcurrency: number) {
+        // Called on a path that runs per task, and the value almost never changes: this
+        // logged "Concurrency updated to 6" 709 times in one day, always to the value it
+        // already had. Log real changes only — a line that fires constantly and says
+        // nothing is what a reader learns to skip past, including on the day it matters.
+        // process() still runs unconditionally: it is idempotent, and a raise that
+        // arrived while the queue was full must be free to dispatch immediately.
+        if (maxTotalConcurrency !== this.maxTotalConcurrency) {
+            logger.debug(`[PriorityQueue] Concurrency changed ${this.maxTotalConcurrency} -> ${maxTotalConcurrency}`);
+        }
         this.maxTotalConcurrency = maxTotalConcurrency;
-        logger.debug(`[PriorityQueue] Concurrency updated to ${maxTotalConcurrency}`);
         this.process();
     }
 

package/src/infrastructure/browser/scheduler-factory.ts

@@ -162,8 +162,19 @@ export async function forceSchedulerRestart(forceClearRemoteState: boolean = fal
 
         logger.log('[Scheduler] Restart complete. Next call will create fresh scheduler.');
     } finally {
+        // Deliberately does NOT dispose the browser-init lock. This function never
+        // acquires it, and FileLockService.dispose() is not a release — it drains for
+        // 5s and then FORCES the teardown, unlinking the lock file and retiring the
+        // instance. A browser init holds that lock for as long as it takes to launch a
+        // browser server, which is routinely longer than 5s, and the retry paths in
+        // task-execution-service call this function mid-run precisely when an init is
+        // likely in flight. The victim's critical section is not cancelled by the
+        // teardown, and the next getBrowserInitLock() builds a fresh instance (new uuid,
+        // new FIFO queue) over the now-unlinked path, so it acquires immediately: two
+        // concurrent browser inits, two elections, two listening servers. The lock is
+        // path-scoped and version-independent, so a restart does not invalidate it;
+        // the owning SchedulerFactoryService disposes it at container teardown.
         schedulerService.setSchedulerRestartInProgress(false);
-        await disposeBrowserInitLock(container).catch(() => {});
     }
 }
 

package/src/infrastructure/browser/task-execution-service.ts

@@ -18,7 +18,7 @@ import { getServiceContainer, getService } from '../../core/service-registry.ts'
 import type { ServiceContainer } from '../../core/service-registry.ts';
 import { ServiceNames } from '../../core/service-interfaces.ts';
 import type { ISchedulerFactory, IScheduler } from '../../core/scheduler-factory.ts';
-import type { ISchedulerInternals } from '../../core/interfaces/scheduler-interfaces.ts';
+import type { ISchedulerInternals, TaskDispatchListener } from '../../core/interfaces/scheduler-interfaces.ts';
 import type { IStateManager } from '../../core/interfaces/state-manager-interfaces.ts';
 import { BrowserClient } from './browser-client.ts';
 
@@ -331,7 +331,7 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
 /**
  * Run a worker search query with retry logic.
  */
-export async function runWorkerSearch(query: string, config?: Config, signal?: AbortSignal, retries = 1, sessionId?: string, container: ServiceContainer = getServiceContainer(), handoverRetries = LEADER_HANDOVER_RETRIES): Promise<SearchResult[]> {
+export async function runWorkerSearch(query: string, config?: Config, signal?: AbortSignal, retries = 1, sessionId?: string, container: ServiceContainer = getServiceContainer(), handoverRetries = LEADER_HANDOVER_RETRIES, onTaskEvent?: TaskDispatchListener): Promise<SearchResult[]> {
     if (signal?.aborted) throw new Error('Aborted');
     
     const breaker = sessionId ? getBrowserCircuitBreaker(sessionId) : browserCircuitBreaker;
@@ -340,7 +340,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
         return await breaker.execute(async () => {
             if (signal?.aborted) throw new Error('Aborted');
             const scheduler = await getScheduler(config, container);
-            return await scheduler.runSearch(query, config, signal);
+            return await scheduler.runSearch(query, config, signal, onTaskEvent);
         });
     } catch (error: unknown) {
         if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw new Error('Aborted', { cause: error });
@@ -358,7 +358,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
             logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
             await recoverFromLeaderHandover(container);
             await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
-            return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
+            return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1, onTaskEvent);
         }
 
         if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
@@ -381,7 +381,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
 
             const jitter = 100 + Math.floor(Math.random() * 400);
             await new Promise(resolve => setTimeout(resolve, jitter));
-            return runWorkerSearch(query, config, signal, retries - 1, sessionId, container, handoverRetries);
+            return runWorkerSearch(query, config, signal, retries - 1, sessionId, container, handoverRetries, onTaskEvent);
         }
         throw error;
     }