@lincoln504/pi-research 1.1.2 → 1.3.0

package/src/infrastructure/browser/browser-task-scheduler.ts

@@ -24,6 +24,52 @@ import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowser
 import { raceWithDeadline } from '../../utils/safe-unref.ts';
 import { PriorityTaskQueue } from './priority-task-queue.ts';
 
+/**
+ * Grace added to the worker's own task budget before the dead-worker backstop
+ * fires, so a live worker's more informative timeout always answers first.
+ */
+export const WORKER_DEATH_BACKSTOP_GRACE_MS = 5000;
+
+/**
+ * Execute a pool task with a dead-worker backstop.
+ *
+ * poolifier (measured on 5.3.2) never settles an in-flight execute() whose worker
+ * dies — not when the worker exits, not when a replacement is respawned, not even
+ * when the pool is destroyed. The worker enforces taskTimeoutMs on itself, but only
+ * while it is alive; an OOM-killed worker leaves the promise pending forever, and
+ * with it the PriorityTaskQueue concurrency slot the dispatched closure holds. Each
+ * such death would permanently shrink the queue until the leader serves nothing but
+ * timeouts.
+ *
+ * The backstop timer is created inside the dispatched closure, so it shares no
+ * lifetime with the caller's outer race — whose `finally` clearing a shared timer
+ * is exactly how a QUEUED task once lost its guard. Armed at dispatch, it fires
+ * only when nothing is left to answer; its rejection frees the queue slot (the
+ * caller has already been answered by the outer race).
+ */
+export async function executePoolTaskWithDeathBackstop<T>(
+    execute: () => Promise<T>,
+    taskTimeoutMs: number,
+    label: string,
+): Promise<T> {
+    const backstopMs = taskTimeoutMs + WORKER_DEATH_BACKSTOP_GRACE_MS;
+    let backstopId: NodeJS.Timeout | undefined;
+    try {
+        return await Promise.race([
+            execute(),
+            new Promise<never>((_, reject) => {
+                backstopId = setTimeout(
+                    () => reject(new Error(`${label} task unanswered after ${backstopMs}ms — worker likely died mid-task`)),
+                    backstopMs,
+                );
+                if (backstopId.unref) backstopId.unref();
+            }),
+        ]);
+    } finally {
+        if (backstopId) clearTimeout(backstopId);
+    }
+}
+
 /**
  * Browser task scheduler - manages the worker pool and executes tasks.
  * Only the leader process has an instance of this scheduler.
@@ -156,9 +202,16 @@ export class BrowserTaskScheduler implements IScheduler {
 
     async startServer(): Promise<number> {
         this.server = new BrowserServer({
-            onSearch: (q) => this.runSearch(q),
-            onScrape: (u) => this.runScrape(u),
-            onHealthCheck: () => this.runHealthCheck(),
+            // The per-request signal aborts when the follower's socket closes
+            // before completion, so the leader-side task releases its queue
+            // claim (abort-while-queued/running is the PriorityTaskQueue's own
+            // pinned semantics) instead of holding the slot to full timeout.
+            // Healthcheck mirrors search/scrape here even though its one current
+            // production caller (src/healthcheck/index.ts) never passes a signal —
+            // the wiring must not silently drop one the moment a caller does.
+            onSearch: (q, signal) => this.runSearch(q, undefined, signal),
+            onScrape: (u, signal) => this.runScrape(u, undefined, signal),
+            onHealthCheck: (signal) => this.runHealthCheck(undefined, signal),
         });
         // FIX (#21): Expose auth secret to child processes via env
         process.env['PI_BROWSER_AUTH_SECRET'] = getBrowserServerAuthSecret();
@@ -192,17 +245,29 @@ export class BrowserTaskScheduler implements IScheduler {
 
         logger.debug(`[BrowserTaskScheduler] Executing search: "${query}" (Timeout: ${timeoutMs}ms)`);
         let result: any;
+        // Settling the outer race must also release the task's queue claim: a task
+        // still QUEUED when the caller is answered is cancelled before it consumes
+        // a slot, and a RUNNING one frees its slot instead of keeping it (plus a
+        // fresh worker budget) for a result nobody will consume. The abort-while-
+        // queued/running semantics are the queue's own, pinned by its tests.
+        const settled = new AbortController();
+        const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
             const queue = this.getPriorityQueue(config);
             // Race the enqueue call against the timeoutPromise.
             // This ensures that even if the queue is saturated, we won't hang forever.
             result = await Promise.race([
                 queue.enqueue('search', async () => {
-                    return await Promise.race([
-                        pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutPromise
-                    ]);
-                }, signal),
+                    // The shared timeoutPromise is deliberately NOT raced in here (its timer
+                    // dies with the outer race); the backstop below carries its own timer, so
+                    // a QUEUED task can never be dispatched against a disarmed guard, and a
+                    // worker dying mid-task cannot pin the queue slot forever.
+                    return await executePoolTaskWithDeathBackstop(
+                        () => pool.execute({ type: 'search', query, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                        timeoutMs,
+                        'Search',
+                    );
+                }, queueSignal),
                 timeoutPromise
             ]);
             logger.debug(`[BrowserTaskScheduler] Search completed: "${query}" in ${Date.now() - startTime}ms`);
@@ -215,6 +280,18 @@ export class BrowserTaskScheduler implements IScheduler {
                 logger.debug(`[BrowserTaskScheduler] Search abandoned during shutdown: "${query}"`);
                 throw error;
             }
+            // The caller's own signal aborting (follower socket closed / research run
+            // cancelled) is what PriorityTaskQueue's abort-while-queued/running
+            // rejection above reports. This is routine, expected cancellation, not a
+            // fault — cancelling a run routinely cancels its in-flight browser tasks,
+            // so this fires often. Checking the signal directly (rather than matching
+            // the queue's rejection text) mirrors how BrowserServer's own catch block
+            // and task-execution-service.ts already classify this. Demote to DEBUG so
+            // it doesn't inflate ERROR counts / browser_search_errors_total.
+            if (signal?.aborted) {
+                logger.debug(`[BrowserTaskScheduler] Search cancelled by caller: "${query}"`);
+                throw error;
+            }
             logger.error(`[BrowserTaskScheduler] Search failed: "${query}"`, error);
             metrics.increment('browser_search_errors_total', 1);
             errorTracker.trackError(error instanceof Error ? error : String(error), {
@@ -226,11 +303,14 @@ export class BrowserTaskScheduler implements IScheduler {
             throw error;
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
+            settled.abort();
         }
 
         const duration = Date.now() - startTime;
-        metrics.observe('browser_search_duration_ms', duration, { status: 'success' });
-        metrics.increment('browser_search_requests_total', 1, { status: 'success' });
+        // In-band worker error checked FIRST: a task the worker reports as failed
+        // must count only as an error. Recording success unconditionally before
+        // this check double-counted every worker-reported failure as
+        // success+error, so dashboards read healthy through a full outage.
         if (result.error) {
             metrics.increment('browser_search_requests_total', 1, { status: 'error' });
             errorTracker.trackError(new Error(result.error), {
@@ -242,6 +322,8 @@ export class BrowserTaskScheduler implements IScheduler {
             });
             throw new Error(result.error);
         }
+        metrics.observe('browser_search_duration_ms', duration, { status: 'success' });
+        metrics.increment('browser_search_requests_total', 1, { status: 'success' });
         return result.results;
     }
 
@@ -265,15 +347,20 @@ export class BrowserTaskScheduler implements IScheduler {
         });
 
         let result: any;
+        // See runSearch: outer settle releases the queue claim; the backstop bounds
+        // a dead worker's never-settling execute().
+        const settled = new AbortController();
+        const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
             const queue = this.getPriorityQueue(config);
             result = await Promise.race([
                 queue.enqueue('scrape', async () => {
-                    return await Promise.race([
-                        pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
-                        timeoutPromise
-                    ]);
-                }, signal),
+                    return await executePoolTaskWithDeathBackstop(
+                        () => pool.execute({ type: 'scrape', url, queuedAt: startTime, taskTimeoutMs: timeoutMs }),
+                        timeoutMs,
+                        'Scrape',
+                    );
+                }, queueSignal),
                 timeoutPromise
             ]);
         } catch (error) {
@@ -283,6 +370,12 @@ export class BrowserTaskScheduler implements IScheduler {
                 logger.debug(`[BrowserTaskScheduler] Scrape abandoned during shutdown: "${url}"`);
                 throw error;
             }
+            // See runSearch: the caller's own signal aborting is routine cancellation
+            // (follower disconnect / run cancelled), not a fault — demote to DEBUG.
+            if (signal?.aborted) {
+                logger.debug(`[BrowserTaskScheduler] Scrape cancelled by caller: "${url}"`);
+                throw error;
+            }
             metrics.increment('browser_scrape_errors_total', 1);
             errorTracker.trackError(error instanceof Error ? error : String(error), {
                 component: 'browser-manager',
@@ -292,11 +385,11 @@ export class BrowserTaskScheduler implements IScheduler {
             throw error;
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
+            settled.abort();
         }
 
         const duration = Date.now() - startTime;
-        metrics.observe('browser_scrape_duration_ms', duration, { status: 'success' });
-        metrics.increment('browser_scrape_requests_total', 1, { status: 'success' });
+        // In-band error first — success-side recording must not precede it (see runSearch).
         if (result.error) {
             metrics.increment('browser_scrape_requests_total', 1, { status: 'error' });
             errorTracker.trackError(new Error(result.error), {
@@ -307,6 +400,8 @@ export class BrowserTaskScheduler implements IScheduler {
             });
             throw new Error(result.error);
         }
+        metrics.observe('browser_scrape_duration_ms', duration, { status: 'success' });
+        metrics.increment('browser_scrape_requests_total', 1, { status: 'success' });
         return result;
     }
 
@@ -324,17 +419,20 @@ export class BrowserTaskScheduler implements IScheduler {
         });
 
         let result: { success: boolean; error?: string };
+        // See runSearch: outer settle releases the queue claim; the backstop bounds
+        // a dead worker's never-settling execute().
+        const settled = new AbortController();
+        const queueSignal = signal ? AbortSignal.any([signal, settled.signal]) : settled.signal;
         try {
             const queue = this.getPriorityQueue(config);
             result = await Promise.race([
                 queue.enqueue('healthcheck', async () => {
-                    const execPromise = pool.execute({ type: 'healthcheck', queuedAt: startTime, taskTimeoutMs: timeoutMs });
-                    execPromise.catch((err: Error) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
-                    return await Promise.race([
-                        execPromise,
-                        timeoutPromise
-                    ]);
-                }, signal),
+                    return await executePoolTaskWithDeathBackstop(() => {
+                        const execPromise = pool.execute({ type: 'healthcheck', queuedAt: startTime, taskTimeoutMs: timeoutMs });
+                        execPromise.catch((err: Error) => logger.debug(`[BrowserTaskScheduler] Background healthcheck task rejection: ${err.message}`));
+                        return execPromise;
+                    }, timeoutMs, 'Healthcheck');
+                }, queueSignal),
                 timeoutPromise
             ]) as { success: boolean; error?: string };
         } catch (error) {
@@ -344,6 +442,13 @@ export class BrowserTaskScheduler implements IScheduler {
                 logger.debug(`[BrowserTaskScheduler] Healthcheck abandoned during shutdown`);
                 throw error;
             }
+            // See runSearch: the caller's own signal aborting is routine cancellation,
+            // not a fault — demote to DEBUG. Reachable now that startServer() forwards
+            // a signal into onHealthCheck (see BrowserServer's onHealthCheck wiring).
+            if (signal?.aborted) {
+                logger.debug(`[BrowserTaskScheduler] Healthcheck cancelled by caller`);
+                throw error;
+            }
             metrics.increment('browser_healthcheck_errors_total', 1);
             errorTracker.trackError(error instanceof Error ? error : String(error), {
                 component: 'browser-manager',
@@ -353,13 +458,14 @@ export class BrowserTaskScheduler implements IScheduler {
             throw error;
         } finally {
             if (timeoutId) clearTimeout(timeoutId);
+            settled.abort();
         }
 
         const duration = Date.now() - startTime;
-        metrics.observe('browser_healthcheck_duration_ms', duration, { status: 'success' });
-        metrics.increment('browser_healthcheck_requests_total', 1, { status: 'success' });
-        metrics.setGauge('browser_pool_health', 1);
-        logger.debug(`[Scheduler] Healthcheck completed in ${duration}ms`);
+        // In-band error first — success-side recording must not precede it (see
+        // runSearch). This path additionally owns the browser_pool_health gauge:
+        // setting it to 1 before the check flickered the pool healthy on every
+        // failed probe during an outage.
         if (result.error) {
             metrics.increment('browser_healthcheck_requests_total', 1, { status: 'error' });
             metrics.setGauge('browser_pool_health', 0);
@@ -371,6 +477,10 @@ export class BrowserTaskScheduler implements IScheduler {
             });
             throw new Error(result.error);
         }
+        metrics.observe('browser_healthcheck_duration_ms', duration, { status: 'success' });
+        metrics.increment('browser_healthcheck_requests_total', 1, { status: 'success' });
+        metrics.setGauge('browser_pool_health', 1);
+        logger.debug(`[Scheduler] Healthcheck completed in ${duration}ms`);
         return result;
     }
 
@@ -411,12 +521,21 @@ export class BrowserTaskScheduler implements IScheduler {
         // reference would be moot. Guarding here keeps shutdown from logging a
         // spurious "Cannot get service 'scheduler' during container disposal".
         if (!this.container.isDisposing) {
-            const schedulerService = await getService<ISchedulerInternals>(ServiceNames.SCHEDULER, undefined, this.container);
-            const currentScheduler = schedulerService.getSchedulerInstance();
-            if (currentScheduler && 'schedulerId' in currentScheduler && currentScheduler.schedulerId === this.schedulerId) {
-                schedulerService.setSchedulerInstance(null);
-                schedulerService.setSchedulerVersion(null);
-                schedulerService.setSchedulerInitializationPromise(null);
+            try {
+                const schedulerService = await getService<ISchedulerInternals>(ServiceNames.SCHEDULER, undefined, this.container);
+                const currentScheduler = schedulerService.getSchedulerInstance();
+                if (currentScheduler && 'schedulerId' in currentScheduler && currentScheduler.schedulerId === this.schedulerId) {
+                    schedulerService.setSchedulerInstance(null);
+                    schedulerService.setSchedulerVersion(null);
+                    schedulerService.setSchedulerInitializationPromise(null);
+                }
+            } catch (err) {
+                // Disposal can begin between the isDisposing check above and the
+                // getService call. isShuttingDown is already latched true, so a
+                // throw here would abort shutdown permanently — drain, state-clear,
+                // server-stop and pool teardown would all be skipped. The reference
+                // being unclearable is the same moot case the guard above covers.
+                logger.debug('[Scheduler] Could not clear scheduler service reference during shutdown:', err);
             }
         }
 
@@ -427,7 +546,12 @@ export class BrowserTaskScheduler implements IScheduler {
             logger.warn('[Scheduler] Could not read browser server state during shutdown:', err);
         }
         if (serverInfo?.pid === process.pid && serverInfo?.schedulerId === this.schedulerId) {
-            await this.stateManager.clearBrowserServer().catch((err) => {
+            // CAS: this belief-check (pid + schedulerId match) happens outside any
+            // lock, so pass it as `expected` rather than relying on it as an
+            // assumption — if a successor claimed the slot between the read above
+            // and this clear, the mismatch makes the clear a no-op instead of
+            // deregistering the new owner.
+            await this.stateManager.clearBrowserServer({ pid: process.pid, schedulerId: this.schedulerId }).catch((err) => {
                 logger.warn('[Scheduler] Failed to clear browser server state during shutdown:', err);
             });
         }

package/src/infrastructure/browser/cleanup-utils.ts

@@ -9,8 +9,9 @@ const DEFAULT_STALE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
  * Firefox/Camoufox profile lock files, cross-platform. A live browser holds:
  *  - Unix:   `lock` — a SYMLINK whose target encodes the owner: "<ip>:+<pid>" or "+<pid>",
  *            plus `.parentlock` (an fcntl-locked empty file).
- *  - Windows: `parent.lock` — a file kept OPEN by the live process (so the OS refuses to
- *            delete it, which is itself a safety net against reclaiming an in-use profile).
+ *  - Windows: `parent.lock` — a file kept OPEN by the live process. That is a liveness
+ *            SIGNAL (see isWindowsParentLockHeld), not a safety net: a recursive delete
+ *            has already removed the rest of the profile before it reaches this file.
  * The previous code checked for `.lock`, which Firefox/Camoufox never creates, so its
  * lock-awareness was inert. These are the real names.
  */
@@ -20,6 +21,37 @@ const LOCK_FILE_NAMES = ['lock', '.parentlock', 'parent.lock'];
  *  so a just-starting concurrent run is not reclaimed out from under itself. */
 const LOCK_FRESHNESS_MS = 2 * 60 * 1000;
 
+/**
+ * Windows: is `parent.lock` still held open by a live browser?
+ *
+ * Firefox opens this file with no sharing, so while the browser runs ANY other
+ * open of it fails with a sharing violation. That makes the probe positive
+ * evidence of a live owner — the same standing as the Unix `lock` symlink's PID.
+ *
+ * Recency cannot substitute for it here: `parent.lock`'s mtime is stamped once at
+ * browser startup and never refreshed, so every browser alive longer than
+ * LOCK_FRESHNESS_MS read as dead — the normal case for a minutes-long research
+ * run. Nor does "the OS refuses to delete a held-open file" save the profile:
+ * `fs.rm({recursive:true})` unlinks entries one at a time, so prefs, cookies and
+ * places are already gone by the time it reaches the locked file and fails.
+ *
+ * Returns null when the answer is not knowable, which callers treat as live —
+ * sparing a leaked profile costs disk, deleting a live one corrupts a running run.
+ */
+async function isWindowsParentLockHeld(lockPath: string): Promise<boolean | null> {
+  let handle: Awaited<ReturnType<typeof fs.open>> | undefined;
+  try {
+    handle = await fs.open(lockPath, 'r');
+  } catch (e) {
+    const code = (e as NodeJS.ErrnoException)?.code;
+    if (code === 'ENOENT') return false; // vanished — no live owner
+    if (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES') return true; // held open
+    return null; // unexpected — don't guess
+  }
+  await handle.close().catch(() => {});
+  return false;
+}
+
 /** Parse the owner PID out of a Firefox `lock` symlink target ("<ip>:+<pid>" / "+<pid>"). */
 function parsePidFromLockTarget(target: string): number | null {
   const m = /\+(\d+)\s*$/.exec(target);
@@ -62,14 +94,62 @@ async function isProfileLive(profileDir: string): Promise<boolean> {
         // unreadable symlink — fall back to freshness below
       }
     }
-    // `.parentlock` / `parent.lock` (no PID encoded), or an unreadable `lock`: use recency
-    // as the liveness proxy. On Windows the file is also held open, so an attempted delete
-    // of a genuinely-live profile fails harmlessly regardless of this heuristic.
+    // Windows has no `lock` symlink, so `parent.lock` is the only signal there. Probe
+    // whether it is still held open — authoritative, unlike recency, which reads every
+    // browser older than LOCK_FRESHNESS_MS as dead because the mtime is never refreshed.
+    if (process.platform === 'win32' && name === 'parent.lock') {
+      const held = await isWindowsParentLockHeld(lockPath);
+      if (held !== null) return held;
+      return true; // indeterminate — spare the profile
+    }
+    // `.parentlock` (no PID encoded) or an unreadable `lock`: use recency as the proxy.
     return Date.now() - lst.mtimeMs < LOCK_FRESHNESS_MS;
   }
   return false;
 }
 
+/** Canonicalize a path for identity comparison: realpath when resolvable (macOS
+ *  /tmp is a symlink to /private/tmp), else a plain resolve; case-folded on
+ *  Windows, whose filesystem compares case-insensitively. */
+async function canonicalizeForCompare(p: string): Promise<string> {
+  let resolved: string;
+  try {
+    resolved = await fs.realpath(p);
+  } catch {
+    resolved = path.resolve(p);
+  }
+  return process.platform === 'win32' ? resolved.toLowerCase() : resolved;
+}
+
+/**
+ * May the unrecognized-entry deletion branch run against this explicit baseDir?
+ *
+ * That branch recursively rm's ANY stale subdirectory, so it is only safe inside
+ * a directory pi-research owns outright. Two conditions gate it: the dir must not
+ * BE the shared system temp dir (realpath-compared on both sides — a
+ * PI_RESEARCH_TMP_DIR pointed at /tmp once made every same-user /tmp entry — ssh
+ * agents, build caches — a sweep candidate), and the path must carry a
+ * pi-research segment as positive evidence of ownership. When this returns false
+ * the prefix-matched playwright profile cleanup still runs; only unknowns are
+ * spared.
+ */
+async function isPiOwnedSweepDir(dir: string): Promise<boolean> {
+  const [realDir, realTmp] = await Promise.all([
+    canonicalizeForCompare(dir),
+    canonicalizeForCompare(os.tmpdir()),
+  ]);
+  if (realDir === realTmp) return false;
+  const hasPiSegment = (p: string) =>
+    p.split(/[\\/]+/).some((segment) => segment.toLowerCase().startsWith('pi-research'));
+  // Only the CANONICAL path counts. A symlinked dir whose target lacks the
+  // marker is NOT sweep-safe: approving it by its configured name would send the
+  // recursive rm THROUGH the symlink into a directory we do not own (e.g. a
+  // /tmp/pi-research/profiles link pre-created by another user under a
+  // world-writable PI_RESEARCH_TMP_DIR). Unknowns are spared; the prefix-matched
+  // playwright profile cleanup still runs either way.
+  return hasPiSegment(realDir);
+}
+
 /**
  * Reclaim leaked Playwright/Camoufox profile directories.
  *
@@ -96,6 +176,11 @@ export async function cleanupStaleProfiles(
   let removed = 0;
   let errors = 0;
 
+  // Explicit-baseDir mode considers EVERY entry a candidate, including ones it
+  // does not recognize; deleting unknowns is only permitted inside a dir that is
+  // provably pi-owned and not the shared system temp dir (see isPiOwnedSweepDir).
+  const sweepUnrecognized = usePrefix ? false : await isPiOwnedSweepDir(scanDir);
+
   try {
     const entries = await fs.readdir(scanDir);
     const now = Date.now();
@@ -127,15 +212,16 @@ export async function cleanupStaleProfiles(
             logger.debug(`[Cleanup] Skipping profile ${entry} — owned by a live browser`);
             continue;
           }
-          await fs.rm(fullPath, { recursive: true, force: true });
+          await fs.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
           removed++;
           continue;
         }
 
         // Unrecognized entry (only reachable with an explicit baseDir): be conservative —
-        // only reclaim if both old AND not live, so we never delete something unknown promptly.
-        if (now - stats.mtimeMs > staleMs && !(await isProfileLive(fullPath))) {
-          await fs.rm(fullPath, { recursive: true, force: true });
+        // only reclaim inside a pi-owned dir, and only if both old AND not live,
+        // so nothing unknown is ever deleted promptly or outside our ownership.
+        if (sweepUnrecognized && now - stats.mtimeMs > staleMs && !(await isProfileLive(fullPath))) {
+          await fs.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
           removed++;
         }
       } catch (_e) {

package/src/infrastructure/browser/client-agent.ts

@@ -10,16 +10,17 @@ import * as http from 'node:http';
 /**
  * Global HTTP Agent for high-concurrency client requests
  *
- * IMPORTANT: The socket timeout must be longer than the BrowserClient request timeout (120s)
- * to prevent premature socket closure when the browser is slow or the queue is deep.
+ * IMPORTANT: The socket timeout must exceed the largest possible BrowserClient
+ * request timeout (CLIENT_TIMEOUT_CAP_MS = 300s in browser-client.ts) so the
+ * request-level timer — whose error text is classified by the retry gates —
+ * always answers before the socket layer tears the connection down.
  */
 const clientAgent = new http.Agent({
     keepAlive: true,
     maxSockets: 100, // Allow up to 100 concurrent requests to the leader
     maxFreeSockets: 10,
-    // Set timeout to 180s (3x the client timeout) to handle slow browser responses
-    // and prevent "socket hang up" errors during peak load
-    timeout: 180000
+    // Above CLIENT_TIMEOUT_CAP_MS (300s) with margin — see the invariant above.
+    timeout: 330000
 });
 
 /**

package/src/infrastructure/browser/config.ts

@@ -61,8 +61,9 @@ export function getBrowserCacheDir(): string {
  * os.tmpdir(): on systems where /tmp is tmpfs (RAM-backed), placing several
  * browser profiles there consumes RAM and, with an OOM killer like earlyoom
  * active, can contribute to the whole session being killed. Override with
- * PI_RESEARCH_TMP_DIR (config.TMP_DIR) — e.g. point it back at the system temp
- * dir to deliberately use tmpfs/RAM when there is memory headroom.
+ * PI_RESEARCH_TMP_DIR (config.TMP_DIR) — e.g. point it at the system temp dir
+ * to deliberately use tmpfs/RAM when there is memory headroom; profiles then
+ * land in <TMP_DIR>/pi-research/profiles, never in TMP_DIR itself.
  *
  * The directory is created if missing, because Playwright requires the parent
  * of its mkdtemp profile dir to already exist.
@@ -71,7 +72,17 @@ export function getBrowserProfileDir(config?: Config): string {
     const configured = (config || getConfig()).TMP_DIR;
     let dir: string;
     if (configured && configured.trim().length > 0) {
-        dir = configured;
+        // Containment invariant: profiles live in a pi-research-owned
+        // SUBDIRECTORY of the configured dir, never the configured dir itself.
+        // The pool-startup sweep (cleanupStaleProfiles with an explicit baseDir)
+        // treats every entry of this directory as a reclaim candidate, so using
+        // PI_RESEARCH_TMP_DIR verbatim — e.g. pointed at /tmp per the tmpfs
+        // opt-in above — put unrelated same-user directories (ssh agents, build
+        // caches) on the sweep's candidate list. The pi-research/profiles shape
+        // also matches PI_BROWSER_MARKER's profile-path fallback in
+        // browser-cleanup.ts, so orphan detection keeps working for relocated
+        // profiles.
+        dir = join(configured, 'pi-research', 'profiles');
     } else {
         const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
         dir = join(cacheHome, 'pi-research', 'profiles');

package/src/infrastructure/browser/priority-task-queue.ts

@@ -149,11 +149,18 @@ export class PriorityTaskQueue {
         // on its own; we swallow its late settlement and free the slot now so the
         // queue can dispatch the next task immediately.
         let onAbort: (() => void) | undefined;
-        const fnPromise = task.fn();
-        // Prevent an unhandled rejection if abort wins the race and fn() rejects later.
-        fnPromise.catch(() => { /* handled via the race below or abandoned on abort */ });
 
         try {
+            // task.fn() is invoked INSIDE the try: a non-async fn that throws
+            // synchronously would otherwise escape before the finally that decrements
+            // activeCount, permanently inflating the active count (starving the queue
+            // by one slot each time) and leaving the caller's enqueue() promise
+            // unsettled. Every fn today is async, so this is a latent shape defect
+            // rather than a live one — but the correct placement costs nothing.
+            const fnPromise = task.fn();
+            // Prevent an unhandled rejection if abort wins the race and fn() rejects later.
+            fnPromise.catch(() => { /* handled via the race below or abandoned on abort */ });
+
             let result: any;
             if (task.signal) {
                 const abortPromise = new Promise<never>((_, rej) => {

package/src/infrastructure/browser/scheduler-factory.ts

@@ -132,7 +132,16 @@ export async function forceSchedulerRestart(forceClearRemoteState: boolean = fal
         }
 
         if (shouldClearState) {
-            await stateManager.clearBrowserServer().catch((error) => {
+            // Compare-and-delete against the registration we just read: without a
+            // lock spanning the read-decide-clear sequence above, a concurrent
+            // process can independently detect the same dead leader, complete its
+            // own (properly locked) election in getScheduler(), and publish a
+            // fresh registration in this exact window. An unconditional clear here
+            // would wipe that brand-new, legitimate leader — it keeps running but
+            // becomes undiscoverable via state, so the next caller elects a
+            // second, redundant leader. See StateBrowserManager.clearBrowserServer.
+            const expected = serverInfo ? { pid: serverInfo.pid, schedulerId: serverInfo.schedulerId } : undefined;
+            await stateManager.clearBrowserServer(expected).catch((error) => {
                 logger.warn('[Scheduler] Failed to clear browser server from state:', error);
             });
         }
@@ -222,12 +231,18 @@ export async function getScheduler(config?: Config, container: ServiceContainer
                         if (serverInfo.pid !== process.pid) {
                             logger.log(`[Scheduler] Bypassing stale scheduler process (PID ${serverInfo.pid}) by clearing state...`);
                         }
-                        await stateManager.clearBrowserServer();
+                        // Defense-in-depth CAS: this read-then-clear is entirely inside
+                        // the browser-init lock, which serializes other getScheduler()
+                        // callers, but callers that clear without taking this lock
+                        // (forceSchedulerRestart, scheduler shutdown) are not excluded —
+                        // pass what was just read so a mismatch no-ops instead of
+                        // deleting a registration this code didn't observe.
+                        await stateManager.clearBrowserServer({ pid: serverInfo.pid, schedulerId: serverInfo.schedulerId });
                     } else {
                         const portOk = await isPortListening(serverInfo.port);
                         if (!portOk) {
                             logger.warn(`[Scheduler] PID ${serverInfo.pid} is alive but port ${serverInfo.port} is not listening — clearing stale state and starting fresh.`);
-                            await stateManager.clearBrowserServer().catch((e) => {
+                            await stateManager.clearBrowserServer({ pid: serverInfo.pid, schedulerId: serverInfo.schedulerId }).catch((e) => {
                                 logger.warn('[Scheduler] Failed to clear stale browser server state:', e);
                             });
                         } else {
@@ -282,6 +297,25 @@ export async function getScheduler(config?: Config, container: ServiceContainer
             let wonElection = false;
             let winnerPort = port;
             let winnerAuthSecret: string | undefined;
+            // Read our start-time BEFORE taking the state lock (the lookup can shell
+            // out on macOS/Windows; getCurrentProcessStartTime caches, so this is a
+            // single cheap read). Recording it alongside the PID is what makes
+            // StateManager.isPidAlive's PID-reuse guard work: that guard passes
+            // `state.browserServer.startTime` as expectedStartTime, so an election
+            // that writes only the PID leaves it permanently undefined and the guard
+            // degrades to a bare signal-0 check — which a recycled PID passes,
+            // making every follower adopt an unrelated process as the browser leader.
+            // Null on a platform where it cannot be read → the entry omits it and
+            // liveness falls back to that bare check, same as pre-existing entries.
+            const myStartTime = await (async () => {
+                try {
+                    const processLifecycle = await getService<IProcessLifecycle>(ServiceNames.PROCESS_LIFECYCLE, undefined, container);
+                    return await processLifecycle.getCurrentProcessStartTime();
+                } catch (err) {
+                    logger.debug('[Scheduler] Could not read own process start time (PID-reuse guard degrades to signal-0):', err);
+                    return null;
+                }
+            })();
             try {
                 await stateManager.updateState(async (state) => {
                     if (state.browserServer) {
@@ -294,7 +328,13 @@ export async function getScheduler(config?: Config, container: ServiceContainer
                         }
                     }
                     const secret = getBrowserServerAuthSecret();
-                    state.browserServer = { port, pid: process.pid, schedulerId, authSecret: secret };
+                    state.browserServer = {
+                        port,
+                        pid: process.pid,
+                        schedulerId,
+                        authSecret: secret,
+                        ...(myStartTime !== null ? { startTime: myStartTime } : {}),
+                    };
                     state.schedulerVersion = schedulerVersion;
                     wonElection = true;
                     return state;