@lincoln504/pi-research 1.3.11 → 1.4.1

package/src/infrastructure/browser/browser-task-scheduler.ts

@@ -19,10 +19,11 @@ import type { IStateManager } from '../../core/interfaces/state-manager-interfac
 import { BrowserServer, getBrowserServerAuthSecret } from './browser-server.ts';
 import { isPoolShutdownError } from './browser-error-utils.ts';
 import type { WorkerPoolManager } from './worker-pool-manager.ts';
-import type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
+import type { IScheduler, TaskDispatchListener } from '../../core/interfaces/scheduler-interfaces.ts';
 import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowserProcesses } from './browser-cleanup.ts';
 import { raceWithDeadline } from '../../utils/safe-unref.ts';
 import { PriorityTaskQueue } from './priority-task-queue.ts';
+import { getHealthCheckBudgetMs } from './config.ts';
 
 /**
  * Grace added to the worker's own task budget before the dead-worker backstop
@@ -74,6 +75,47 @@ export async function executePoolTaskWithDeathBackstop<T>(
  * Browser task scheduler - manages the worker pool and executes tasks.
  * Only the leader process has an instance of this scheduler.
  */
+/**
+ * How long a task actually RAN, given when it was dispatched and when it was
+ * submitted.
+ *
+ * Duration used to be `now - submittedAt`, which folds queue wait into the work.
+ * That is the number `Search completed ... in Nms` prints and the number
+ * `browser_search_duration_ms` records, so it is what every latency investigation
+ * reads — and during a saturated burst it reported a 51s median against a true
+ * 5.8s execution median. A whole evening went into explaining a slowdown that had
+ * not happened. Moving the deadline off queue wait while leaving the measurement
+ * on it would have preserved exactly that misdiagnosis.
+ *
+ * A task that never reached a worker has no execution time; report its full
+ * elapsed span rather than zero, so a timeout is not silently recorded as instant.
+ */
+export function executionMs(dispatchedAt: number, submittedAt: number, now: number = Date.now()): number {
+    return dispatchedAt > 0 ? now - dispatchedAt : now - submittedAt;
+}
+
+/** How long a task sat in the queue before a worker took it. Zero if never dispatched. */
+export function queueWaitMs(dispatchedAt: number, submittedAt: number): number {
+    return dispatchedAt > 0 ? dispatchedAt - submittedAt : 0;
+}
+
+/**
+ * Record a task's queue wait, whether or not it was ever dispatched.
+ *
+ * Split by outcome because the two are different quantities and must not be summed
+ * into one distribution: a dispatched task's wait ENDED, while an abandoned task's
+ * wait was still running when the caller gave up, so the latter is a lower bound.
+ * Folding abandonments in as `queueWaitMs`'s zero would pull the distribution
+ * DOWNWARD in exactly the bursts where it should spike.
+ */
+export function observeQueueWait(metric: string, dispatchedAt: number, submittedAt: number): void {
+    metrics.observe(
+        metric,
+        dispatchedAt > 0 ? dispatchedAt - submittedAt : Date.now() - submittedAt,
+        { dispatched: dispatchedAt > 0 ? 'yes' : 'no' },
+    );
+}
+
 export class BrowserTaskScheduler implements IScheduler {
     private workerPoolManager: WorkerPoolManager | null = null;
     private server: BrowserServer | null = null;
@@ -169,6 +211,9 @@ export class BrowserTaskScheduler implements IScheduler {
                 // Decay (not reset) so a slow worker crash-loop can still accumulate to
                 // the auto-recovery threshold instead of being zeroed on every tick.
                 poolManager.decayConsecutiveErrors();
+                // Sampled on the leadership tick rather than per task: this is a
+                // distribution property, not a per-task one, and it should read zero.
+                poolManager.recordDispatchHealth();
             } catch (err) {
                 logger.warn('[Scheduler] Leadership check error:', err);
             } finally {
@@ -218,7 +263,7 @@ export class BrowserTaskScheduler implements IScheduler {
         return this.server.start();
     }
 
-    async runSearch(query: string, config?: Config, signal?: AbortSignal): Promise<SearchResult[]> {
+    async runSearch(query: string, config?: Config, signal?: AbortSignal, onTaskEvent?: TaskDispatchListener): Promise<SearchResult[]> {
         // Reject tasks once this scheduler has begun teardown. shutdown() nulls priorityQueue and
         // clears the scheduler reference (a future run gets a fresh scheduler), so getPriorityQueue()
         // here would otherwise build a FRESH, non-shutdown queue on this dead instance and dispatch
@@ -235,12 +280,36 @@ export class BrowserTaskScheduler implements IScheduler {
         const cfg = config || getConfig();
         const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
 
+        // The deadline measures EXECUTION, not queue wait: the timer is armed by
+        // `startDeadline()` from inside the enqueued callback, i.e. once the queue has
+        // actually dispatched this task to a worker.
+        //
+        // It used to be armed here, before enqueuing, so a saturated queue burned a task's
+        // entire budget waiting for a slot and it aborted having done no work at all. That
+        // made the number of tasks a burst could complete `capacity x (budget / latency)`
+        // rather than the number submitted — measured on a live run, 78 of 100 searches
+        // aborted at the deadline without ever running. A queue is supposed to absorb
+        // volume by making work wait, not by killing whatever it could not reach in time.
+        // Waiting is now bounded by the caller's signal and the run-level timeouts; a task
+        // that does get dispatched always gets its full budget.
+        // `dispatchedAt` is the same instant, kept for MEASUREMENT rather than for the
+        // deadline. Reporting `Date.now() - startTime` as the search's duration charged
+        // queue wait to the work, and that number is what every latency analysis reads:
+        // on the night this was diagnosed it reported a 51s median against a true 5.8s
+        // execution median, and invented an eightfold latency regression that never
+        // happened. Moving the deadline off queue wait without moving the measurement
+        // would have left the misdiagnosis in place while claiming to have fixed it.
         let timeoutId: NodeJS.Timeout | undefined;
+        let dispatchedAt = 0;
+        let startDeadline: () => void = () => {};
         const timeoutPromise = new Promise<never>((_, reject) => {
-            timeoutId = setTimeout(() => {
-                reject(new Error(`Search task timed out after ${timeoutMs}ms (including queue wait). query="${query}"`));
-            }, timeoutMs);
-            if (timeoutId.unref) timeoutId.unref();
+            startDeadline = () => {
+                dispatchedAt = Date.now();
+                timeoutId = setTimeout(() => {
+                    reject(new Error(`Search task timed out after ${timeoutMs}ms. query="${query}"`));
+                }, timeoutMs);
+                if (timeoutId.unref) timeoutId.unref();
+            };
         });
 
         logger.debug(`[BrowserTaskScheduler] Executing search: "${query}" (Timeout: ${timeoutMs}ms)`);
@@ -258,19 +327,24 @@ export class BrowserTaskScheduler implements IScheduler {
             // This ensures that even if the queue is saturated, we won't hang forever.
             result = await Promise.race([
                 queue.enqueue('search', async () => {
-                    // The shared timeoutPromise is deliberately NOT raced in here (its timer
-                    // dies with the outer race); the backstop below carries its own timer, so
-                    // a QUEUED task can never be dispatched against a disarmed guard, and a
-                    // worker dying mid-task cannot pin the queue slot forever.
-                    return await executePoolTaskWithDeathBackstop(
-                        () => pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutMs,
-                        'Search',
-                    );
+                    startDeadline();
+                    // Same instant, for the same reason, one layer up: whoever called us
+                    // arms their own guard here instead of at call time, and disarms it
+                    // when we settle. A listener that throws must not take the task down.
+                    try { onTaskEvent?.('dispatched'); } catch { /* caller's guard, caller's problem */ }
+                    try {
+                        return await executePoolTaskWithDeathBackstop(
+                            () => pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                            timeoutMs,
+                            'Search',
+                        );
+                    } finally {
+                        try { onTaskEvent?.('settled'); } catch { /* as above */ }
+                    }
                 }, queueSignal),
                 timeoutPromise
             ]);
-            logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${Date.now() - startTime}ms`);
+            logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${executionMs(dispatchedAt, startTime)}ms (queued ${queueWaitMs(dispatchedAt, startTime)}ms)`);
         } catch (error) {
             // A pool-shutdown/destroy error means the run is being torn down (quit/SIGTERM
             // mid-search-burst): the task never ran because dispose() destroyed the pool.
@@ -304,9 +378,16 @@ export class BrowserTaskScheduler implements IScheduler {
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
             settled.abort();
+            // Recorded here rather than on the success path below, because the tasks
+            // that wait longest are precisely the ones that do not succeed — measuring
+            // queue wait only on completed tasks censors the tail this metric exists to
+            // expose. A task that never reached a worker has no dispatch instant, so its
+            // wait is the whole elapsed time and is labelled as such; averaging it in as
+            // a zero would have understated saturation exactly when it was worst.
+            observeQueueWait('browser_search_queue_wait_ms', dispatchedAt, startTime);
         }
 
-        const duration = Date.now() - startTime;
+        const duration = executionMs(dispatchedAt, startTime);
         // In-band worker error checked FIRST: a task the worker reports as failed
         // must count only as an error. Recording success unconditionally before
         // this check double-counted every worker-reported failure as
@@ -327,7 +408,7 @@ export class BrowserTaskScheduler implements IScheduler {
         return result.results;
     }
 
-    async runScrape(url: string, config?: Config, signal?: AbortSignal): Promise<ScrapeResult> {
+    async runScrape(url: string, config?: Config, signal?: AbortSignal, onTaskEvent?: TaskDispatchListener): Promise<ScrapeResult> {
         if (this.isShuttingDown) throw new Error('Worker pool is shutting down');
         this.resetIdleTimer();
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
@@ -338,12 +419,19 @@ export class BrowserTaskScheduler implements IScheduler {
         const isMocking = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
         const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS);
 
+        // Armed on dispatch, not on enqueue — see runSearch for why.
+        // See runSearch: dispatch time is tracked for measurement, not for the deadline.
         let timeoutId: NodeJS.Timeout | undefined;
+        let dispatchedAt = 0;
+        let startDeadline: () => void = () => {};
         const timeoutPromise = new Promise<never>((_, reject) => {
-            timeoutId = setTimeout(() => {
-                reject(new Error(`Scrape task timed out after ${timeoutMs}ms (including queue wait). url="${url}"`));
-            }, timeoutMs);
-            if (timeoutId.unref) timeoutId.unref();
+            startDeadline = () => {
+                dispatchedAt = Date.now();
+                timeoutId = setTimeout(() => {
+                    reject(new Error(`Scrape task timed out after ${timeoutMs}ms. url="${url}"`));
+                }, timeoutMs);
+                if (timeoutId.unref) timeoutId.unref();
+            };
         });
 
         let result: any;
@@ -355,11 +443,18 @@ export class BrowserTaskScheduler implements IScheduler {
             const queue = this.getPriorityQueue(config);
             result = await Promise.race([
                 queue.enqueue('scrape', async () => {
-                    return await executePoolTaskWithDeathBackstop(
-                        () => pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutMs,
-                        'Scrape',
-                    );
+                    startDeadline();
+                    // See runSearch: the caller's guard is armed on dispatch and disarmed on settle.
+                    try { onTaskEvent?.('dispatched'); } catch { /* caller's guard, caller's problem */ }
+                    try {
+                        return await executePoolTaskWithDeathBackstop(
+                            () => pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                            timeoutMs,
+                            'Scrape',
+                        );
+                    } finally {
+                        try { onTaskEvent?.('settled'); } catch { /* as above */ }
+                    }
                 }, queueSignal),
                 timeoutPromise
             ]);
@@ -386,9 +481,11 @@ export class BrowserTaskScheduler implements IScheduler {
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
             settled.abort();
+            // See runSearch: recorded on every outcome, not only on success.
+            observeQueueWait('browser_scrape_queue_wait_ms', dispatchedAt, startTime);
         }
 
-        const duration = Date.now() - startTime;
+        const duration = executionMs(dispatchedAt, startTime);
         // In-band error first — success-side recording must not precede it (see runSearch).
         if (result.error) {
             metrics.increment('browser_scrape_requests_total', 1, { status: 'error' });
@@ -405,17 +502,103 @@ export class BrowserTaskScheduler implements IScheduler {
         return result;
     }
 
-    async runHealthCheck(config?: Config, signal?: AbortSignal): Promise<{ success: boolean }> {
+    async runHealthCheck(config?: Config, signal?: AbortSignal, budgetOverrideMs?: number): Promise<{ success: boolean }> {
         if (this.isShuttingDown) throw new Error('Worker pool is shutting down');
         this.resetIdleTimer();
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
         const startTime = Date.now();
         const isMocking = process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' || process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
-        const timeoutMs = (45000 + 60000) / (isMocking ? 4 : 1);
+        // Derived, not hardcoded — see getHealthCheckBudgetMs. The wait-phase verdict
+        // below rests on this budget out-waiting any task that can hold a slot, and a
+        // fixed 105s made that true only for the default timeouts. `budgetOverrideMs`
+        // exists so the verdict is reachable from a test at all: at 105s it is not,
+        // which is exactly how a previous version of it shipped unreachable and stayed
+        // that way.
+        const timeoutMs = budgetOverrideMs ?? getHealthCheckBudgetMs(config) / (isMocking ? 4 : 1);
+
+        // A probe that never got a worker used to fail as "timed out (including queue
+        // wait)", and the registry reported BrowserRuntime CRITICALLY UNHEALTHY —
+        // during a burst in which the pool was healthily executing the run's own
+        // searches. Twice on the night this was measured.
+        //
+        // Arming the deadline on DISPATCH is the whole fix, and it is sufficient. Three
+        // attempts were made to add a second, wait-phase verdict that would excuse a
+        // saturated pool, and all three were wrong. They are recorded here because each
+        // one reads sensibly and the next reader will otherwise reach for one of them:
+        //
+        //   1. "did the queue dispatch anything recently" — unreachable. Healthcheck
+        //      tasks have strict queue priority, so if a slot frees while this probe
+        //      waits, the queue dispatches THIS probe and cancels the timer. The timer
+        //      can only fire in a window where nothing dispatched at all.
+        //   2. "are the slots occupied" — wrong in the dangerous direction. activeCount
+        //      rises before pool.execute, so six workers hung inside a browser launch
+        //      are six occupied slots, and the exact incident this was written for
+        //      would have reported HEALTHY.
+        //   3. "has a task completed successfully recently" — unreachable for the same
+        //      reason as (1). A success frees a slot, a freed slot dispatches this
+        //      probe, so at expiry the last success is necessarily older than the whole
+        //      budget. The condition it tested was false by construction.
+        //
+        // The general form of the mistake: with strict priority, a probe that waited
+        // out its entire budget without being dispatched IS the evidence of a wedge.
+        // There is no second signal to consult, and every attempt to find one either
+        // could not fire or fired for a dead pool. The numbers below are kept for the
+        // DIAGNOSTIC message — an operator needs to know whether the slots were busy
+        // and when work last succeeded — and not for the verdict.
+        //
+        // False positives are bounded by arithmetic rather than by judgement: the probe
+        // allows 105s while every task that can hold a slot is itself deadlined well
+        // under that (search 55s, scrape 25s plus backstop), so a slot frees inside the
+        // budget unless the pool has genuinely stopped.
+        //
+        // Once dispatched, the deadline measures execution alone (armed below, like
+        // runSearch/runScrape), so a slow probe still fails on its own merits.
         let timeoutId: NodeJS.Timeout | undefined;
+        let dispatched = false;
+        let hcDispatchedAt = 0;
+        let startDeadline: () => void = () => {};
+        // Captured BEFORE the timer can run. Resolving the queue inside the callback
+        // would call getPriorityQueue() on a scheduler that may have shut down in the
+        // meantime, which builds a FRESH queue on the dead instance — undoing the
+        // isShutdown latch that exists to reject enqueues racing teardown, and
+        // reporting a routine shutdown as a wedge because a new queue has dispatched
+        // nothing.
+        const healthQueue = this.getPriorityQueue(config);
+        // Rejects only. It used to be typed as resolvable because a branch here answered
+        // "healthy" without running the probe; nothing does that now, and the type says so.
         const timeoutPromise = new Promise<never>((_, reject) => {
-            timeoutId = setTimeout(() => reject(new Error(`Health check timed out after ${timeoutMs}ms (including queue wait)`)), timeoutMs);
+            const onWaitExpired = () => {
+                if (dispatched) return;
+                const lastSuccessAt = healthQueue.getLastSuccessAt();
+                const sinceSuccess = lastSuccessAt > 0 ? Date.now() - lastSuccessAt : Number.POSITIVE_INFINITY;
+                // The wording is load-bearing, not prose. isBusyPoolHealthFailure
+                // (src/healthcheck/index.ts) decides whether a failed probe aborts the
+                // run, and it decides by matching /tim(e|ed) out|timeout/ on this very
+                // string. An earlier rewrite of this message dropped the word entirely —
+                // so a probe that queued out under load stopped reading as a timeout, the
+                // tolerance gate rejected it, and a BUSY pool aborted the run at
+                // quick-research-orchestrator and research-health. The commit that did it
+                // was the one meant to stop a busy pool being reported as dead; it moved
+                // the abort one layer up. Pinned by a test that drives this exact string
+                // through that exact classifier.
+                //
+                // The rest is diagnosis: slots busy with nothing completing is a worker
+                // fault, slots free is a queue fault (the queue stopped dispatching), and
+                // the last-success age separates "stopped just now" from "never started".
+                reject(new Error(
+                    `Health check timed out after ${timeoutMs}ms without reaching a worker — ${healthQueue.getActiveCount()} of ${healthQueue.getConcurrency()} slots busy, ` +
+                    `${lastSuccessAt > 0 ? `last successful task ${sinceSuccess}ms ago` : 'no task has ever completed successfully'}.`,
+                ));
+            };
+            timeoutId = setTimeout(onWaitExpired, timeoutMs);
             if (timeoutId.unref) timeoutId.unref();
+            startDeadline = () => {
+                dispatched = true;
+                hcDispatchedAt = Date.now();
+                clearTimeout(timeoutId);
+                timeoutId = setTimeout(() => reject(new Error(`Health check timed out after ${timeoutMs}ms.`)), timeoutMs);
+                if (timeoutId.unref) timeoutId.unref();
+            };
         });
 
         let result: { success: boolean; error?: string };
@@ -424,9 +607,11 @@ export class BrowserTaskScheduler implements IScheduler {
         const settled = new AbortController();
         const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
-            const queue = this.getPriorityQueue(config);
+            // Same instance the wait-expiry message reads, so the diagnosis cannot
+            // describe a different queue from the one this probe waited on.
             result = await Promise.race([
-                queue.enqueue('healthcheck', async () => {
+                healthQueue.enqueue('healthcheck', async () => {
+                    startDeadline();
                     return await executePoolTaskWithDeathBackstop(() => {
                         const execPromise = pool.execute({ type: 'healthcheck', queuedAt: startTime, taskTimeoutMs: timeoutMs });
                         execPromise.catch((err: Error) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
@@ -459,9 +644,10 @@ export class BrowserTaskScheduler implements IScheduler {
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
             settled.abort();
+            observeQueueWait('browser_healthcheck_queue_wait_ms', hcDispatchedAt, startTime);
         }
 
-        const duration = Date.now() - startTime;
+        const duration = executionMs(hcDispatchedAt, startTime);
         // In-band error first — success-side recording must not precede it (see
         // runSearch). This path additionally owns the browser_pool_health gauge:
         // setting it to 1 before the check flickered the pool healthy on every

package/src/infrastructure/browser/config.ts

@@ -31,15 +31,40 @@ function getWindowsCamoufoxDir(): string {
     return join(homedir(), 'AppData', 'Local', 'camoufox', 'camoufox', 'Cache');
 }
 
+/**
+ * The custom camoufox location, if the user set one.
+ *
+ * CAMOUFOX_INSTALL_DIR takes precedence because it is camoufox-js's OWN
+ * variable and the only one that can ever move the install: camoufox-js ≥0.12.0
+ * resolves INSTALL_DIR from it. PLAYWRIGHT_BROWSERS_PATH is kept as a documented
+ * alias, but on its own it only ever moved where we LOOK — camoufox went on
+ * installing to the default cache, so setting it made the browser permanently
+ * "not found" right after a successful download. Both are exported onward to
+ * camoufox-js (getBrowserEnv, scripts/setup.cjs) so that install and lookup
+ * cannot disagree.
+ *
+ * CAVEAT, deliberate: the pinned camoufox-js (0.10.x — see
+ * thread-worker-browser.ts for why it cannot move yet) hardcodes
+ * `userCacheDir("camoufox")` and honours NEITHER variable, so on the shipped
+ * pin a custom location still only relocates lookup and the underlying bug
+ * remains. This plumbing is correct and inert rather than wrong: it starts
+ * working the moment the pin can advance, and costs nothing until then. Until
+ * that happens, treat a custom browser directory as unsupported.
+ */
+function getCustomCamoufoxDir(): string | undefined {
+    return process.env['CAMOUFOX_INSTALL_DIR'] || process.env['PLAYWRIGHT_BROWSERS_PATH'] || undefined;
+}
+
 /**
  * Get the camoufox binary cache directory.
- * Uses PLAYWRIGHT_BROWSERS_PATH if set, otherwise the standard user cache location
+ * Uses the custom location above if set, otherwise the standard user cache location
  * that camoufox uses by default (~/.cache/camoufox on Linux).
  * We do NOT override HOME — that trick was unreliable and caused install/runtime mismatches.
  */
 export function getBrowserCacheDir(): string {
-    if (process.env['PLAYWRIGHT_BROWSERS_PATH']) {
-        return process.env['PLAYWRIGHT_BROWSERS_PATH'];
+    const custom = getCustomCamoufoxDir();
+    if (custom) {
+        return custom;
     }
     // Mirror getCamoufoxBinaryPath() so both functions agree on the cache location
     const osPlatform = platform();
@@ -109,11 +134,17 @@ export function getBrowserProfileDir(config?: Config): string {
  */
 export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
     const env: NodeJS.ProcessEnv = { ...process.env };
-    const customPath = process.env['PLAYWRIGHT_BROWSERS_PATH'];
+    const customPath = getCustomCamoufoxDir();
     if (customPath) {
         env['PLAYWRIGHT_BROWSERS_PATH'] = customPath;
+        // The one that camoufox-js actually reads. Without it the worker's
+        // camoufox launches from the DEFAULT cache while everything on this side
+        // resolves the custom path — the install/runtime mismatch this file's
+        // header warns about, just arriving through a different door.
+        env['CAMOUFOX_INSTALL_DIR'] = customPath;
     } else {
         delete env['PLAYWRIGHT_BROWSERS_PATH'];
+        delete env['CAMOUFOX_INSTALL_DIR'];
     }
     // Tell thread-workers where to write their lifecycle/error log. Precedence:
     //   1. A user-set PI_RESEARCH_LOG_FILE (already copied from process.env above) wins —
@@ -164,7 +195,7 @@ export function ensureBrowserCacheDir(): string {
  * Matches camoufox-js's own resolution logic.
  */
 export function getCamoufoxBinaryPath(): string {
-    const customPath = process.env['PLAYWRIGHT_BROWSERS_PATH'];
+    const customPath = getCustomCamoufoxDir();
     if (customPath) {
         return customPath;
     }
@@ -296,4 +327,44 @@ export function isBrowserAvailable(): boolean {
 function isFullMockMode(): boolean {
     return process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' &&
            process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
-}
+}
+/**
+ * How long the leader's health probe may take, end to end.
+ *
+ * This was a hardcoded 105s (45s probe + 60s queue-wait margin) and the verdict logic
+ * that reads it is built on an arithmetic claim: the probe out-waits any task that can
+ * hold a worker slot, so a probe that never reaches a worker really has found a wedge.
+ * That claim held for the DEFAULT timeouts and for no others. `SEARCH_TIMEOUT_MS` and
+ * `BROWSER_TASK_TIMEOUT_MS` are both settable to 120 000, so a search may legally hold
+ * a slot for 240s — and a user who merely raised the search timeout would see healthy
+ * pools reported as wedged, which aborts the run at the readiness gate.
+ *
+ * Deriving it instead makes the invariant hold by construction:
+ *   - the longest a single task may hold a slot, plus
+ *   - the probe's own work: the worker runs up to THREE navigation attempts (primary,
+ *     retry, neutral-endpoint fallback) at `max(10s, HEALTH_CHECK_TIMEOUT_MS)` each,
+ *     and a budget below that sum makes the fallback ladder unreachable — so raising
+ *     HEALTH_CHECK_TIMEOUT_MS used to make the health check strictly worse.
+ *
+ * Floored at the historical 105s so default installs are unchanged.
+ */
+export const HEALTHCHECK_MIN_BUDGET_MS = 45_000 + 60_000;
+
+/** Navigation attempts executeHealthCheck makes: primary, retry, fallback endpoint. */
+const HEALTHCHECK_NAV_ATTEMPTS = 3;
+
+export function getHealthCheckBudgetMs(config?: Config): number {
+    const c = (config || getConfig()) as Partial<Config>;
+    // Each field is read defensively. A missing or non-finite value would otherwise
+    // propagate a NaN into `setTimeout`, which Node coerces to 1ms — turning the health
+    // probe's deadline into an instant failure. Falling back to the schema default is
+    // the only reading that cannot silently invert the bound.
+    const ms = (value: unknown, fallback: number): number =>
+        typeof value === 'number' && Number.isFinite(value) ? value : fallback;
+    const longestSlotHold =
+        Math.max(ms(c.SEARCH_TIMEOUT_MS, 45_000), ms(c.SCRAPE_TIMEOUT_MS, 15_000))
+        + ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000);
+    // Mirrors the worker's own floor in executeHealthCheck.
+    const probeWork = HEALTHCHECK_NAV_ATTEMPTS * Math.max(10_000, ms(c.HEALTH_CHECK_TIMEOUT_MS, 10_000));
+    return Math.max(HEALTHCHECK_MIN_BUDGET_MS, longestSlotHold + probeWork);
+}

package/src/infrastructure/browser/priority-task-queue.ts

@@ -2,6 +2,20 @@ import { logger } from '../../logger.ts';
 
 export type TaskType = 'search' | 'scrape' | 'healthcheck';
 
+/**
+ * Did this resolved value carry an in-band worker failure?
+ *
+ * Worker tasks report failure by RESOLVING with `{ error: ... }` rather than by
+ * rejecting, so "the promise settled successfully" and "the task succeeded" are
+ * different facts about the same value.
+ */
+function isTaskFailureResult(result: unknown): boolean {
+    return typeof result === 'object'
+        && result !== null
+        && typeof (result as { error?: unknown }).error === 'string'
+        && (result as { error: string }).error.length > 0;
+}
+
 export interface QueuedTask<T> {
     type: TaskType;
     fn: () => Promise<T>;
@@ -27,6 +41,16 @@ export class PriorityTaskQueue {
     private maxTotalConcurrency: number;
     private readonly maxQueueDepth: number;
     private isShutdown = false;
+    /** When a task last completed WITHOUT error (epoch ms), or 0 if never.
+     *
+     *  Diagnostic only — it is reported in the health probe's failure message so an
+     *  operator can tell a pool that stopped a moment ago from one that never started.
+     *  It was briefly used as a health VERDICT and must not be again: a success frees a
+     *  slot, a freed slot dispatches the waiting probe (healthchecks have strict
+     *  priority), so by the time a probe times out this value is necessarily older than
+     *  its whole budget. Failures deliberately do not count either — a wedged pool still
+     *  turns its slots over as deadlines and death-backstops fire. */
+    private lastSuccessAt = 0;
 
     constructor(maxTotalConcurrency: number, maxQueueDepth = 500) {
         this.maxTotalConcurrency = maxTotalConcurrency;
@@ -90,6 +114,21 @@ export class PriorityTaskQueue {
         });
     }
 
+    /** Epoch ms of the last task that completed successfully, or 0 if none has. */
+    getLastSuccessAt(): number {
+        return this.lastSuccessAt;
+    }
+
+    /** Tasks currently executing on a worker. */
+    getActiveCount(): number {
+        return this.activeCount;
+    }
+
+    /** How many tasks may execute at once. */
+    getConcurrency(): number {
+        return this.maxTotalConcurrency;
+    }
+
     private removeFromQueue(task: QueuedTask<any>): boolean {
         const queues = [this.healthcheckQueue, this.searchQueue, this.scrapeQueue];
         for (const q of queues) {
@@ -171,6 +210,13 @@ export class PriorityTaskQueue {
             } else {
                 result = await fnPromise;
             }
+            // A worker-reported failure arrives as a RESOLVED value carrying `error` —
+            // poolifier delivers it that way, and the scheduler only turns it into a
+            // throw afterwards. Resolution alone therefore does not mean the task
+            // succeeded: a worker returning `{error:'Task timed out'}` for every task
+            // would keep this timestamp fresh through a total outage, which is the one
+            // thing it must not do.
+            if (!isTaskFailureResult(result)) this.lastSuccessAt = Date.now();
             task.resolve(result);
         } catch (err) {
             task.reject(err);
@@ -187,8 +233,16 @@ export class PriorityTaskQueue {
      * Update the maximum concurrency limit (e.g. if config changes).
      */
     updateConcurrency(maxTotalConcurrency: number) {
+        // Called on a path that runs per task, and the value almost never changes: this
+        // logged "Concurrency updated to 6" 709 times in one day, always to the value it
+        // already had. Log real changes only — a line that fires constantly and says
+        // nothing is what a reader learns to skip past, including on the day it matters.
+        // process() still runs unconditionally: it is idempotent, and a raise that
+        // arrived while the queue was full must be free to dispatch immediately.
+        if (maxTotalConcurrency !== this.maxTotalConcurrency) {
+            logger.debug(`[PriorityQueue] Concurrency changed ${this.maxTotalConcurrency} -> ${maxTotalConcurrency}`);
+        }
         this.maxTotalConcurrency = maxTotalConcurrency;
-        logger.debug(`[PriorityQueue] Concurrency updated to ${maxTotalConcurrency}`);
         this.process();
     }
 

package/src/infrastructure/browser/scheduler-factory.ts

@@ -162,8 +162,19 @@ export async function forceSchedulerRestart(forceClearRemoteState: boolean = fal
 
         logger.log('[Scheduler] Restart complete. Next call will create fresh scheduler.');
     } finally {
+        // Deliberately does NOT dispose the browser-init lock. This function never
+        // acquires it, and FileLockService.dispose() is not a release — it drains for
+        // 5s and then FORCES the teardown, unlinking the lock file and retiring the
+        // instance. A browser init holds that lock for as long as it takes to launch a
+        // browser server, which is routinely longer than 5s, and the retry paths in
+        // task-execution-service call this function mid-run precisely when an init is
+        // likely in flight. The victim's critical section is not cancelled by the
+        // teardown, and the next getBrowserInitLock() builds a fresh instance (new uuid,
+        // new FIFO queue) over the now-unlinked path, so it acquires immediately: two
+        // concurrent browser inits, two elections, two listening servers. The lock is
+        // path-scoped and version-independent, so a restart does not invalidate it;
+        // the owning SchedulerFactoryService disposes it at container teardown.
         schedulerService.setSchedulerRestartInProgress(false);
-        await disposeBrowserInitLock(container).catch(() => {});
     }
 }