@lincoln504/pi-research 1.3.12 → 1.4.1

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -259,15 +259,42 @@ async function extractSearchResults(page: any): Promise<any[]> {
 /**
  * Execute a search task
  */
+/**
+ * Bind a task-timeout signal to the page it is bounding.
+ *
+ * `page.setDefaultTimeout` bounds Playwright's action and navigation verbs
+ * individually; it does not bound their sum, and it does not bound a Response read
+ * at all. Closing the page is what actually ends the work: any in-flight call
+ * rejects with 'Target closed', which runTask's catch already maps back to the
+ * task timeout. Fire-and-forget, `{ once }` so the listener cannot outlive the task.
+ */
+function bindTaskAbortToPage(page: any, signal?: AbortSignal): void {
+  if (!signal) return;
+  const onAbort = () => { page.close().catch(() => {}); };
+  if (signal.aborted) onAbort();
+  else signal.addEventListener('abort', onAbort, { once: true });
+}
+
 export async function executeSearchTask(
   _context: any,
-  query: string
+  query: string,
+  signal?: AbortSignal
 ): Promise<{ results: any[]; jitter: number }> {
   const page = await createPageSafe(_context);
   const SEARCH_TIMEOUT = parseTimeoutMs(process.env['PI_RESEARCH_SEARCH_TIMEOUT_MS'], 45000);
   page.setDefaultTimeout(SEARCH_TIMEOUT);
   page.setDefaultNavigationTimeout(SEARCH_TIMEOUT);
 
+  // Same binding the scrape path has used all along, and for the same reason: the
+  // worker's task deadline was armed but its signal was only ever handed to
+  // executeScrapeTask, so on a search it fired into nothing. Playwright's own
+  // per-action timeouts still bound each verb, but the sum of them plus the jitter
+  // sleep can outlast the whole task budget — and poolifier keeps counting this node
+  // as executing until the worker replies, so the node stays pinned long after the
+  // caller was answered. Closing the page rejects whatever is in flight with
+  // 'Target closed', which runTask's catch already recognises as the timeout.
+  bindTaskAbortToPage(page, signal);
+
   try {
     logToDebugFile('DEBUG', `[Worker-${workerId}] Starting search for: ${query}`);
     
@@ -692,11 +719,15 @@ const HEALTH_FALLBACK_URL = 'https://example.com/';
 async function executeHealthCheckAttempt(
   _context: any,
   navTimeoutMs: number,
-  url: string = HEALTH_PRIMARY_URL
+  url: string = HEALTH_PRIMARY_URL,
+  signal?: AbortSignal
 ): Promise<{ success: boolean; navMs: number }> {
   const page = await createPageSafe(_context);
   page.setDefaultTimeout(navTimeoutMs);
   page.setDefaultNavigationTimeout(navTimeoutMs);
+  // See executeSearchTask: the probe runs twice (primary then fallback), so its own
+  // nav timeout bounds each attempt but not the pair.
+  bindTaskAbortToPage(page, signal);
 
   try {
     const navStart = Date.now();
@@ -721,7 +752,8 @@ async function executeHealthCheckAttempt(
  */
 export async function executeHealthCheck(
   _context: any,
-  initMs: number
+  initMs: number,
+  signal?: AbortSignal
 ): Promise<{ success: boolean; navMs: number }> {
   // Nav timeout: read from env (passed through getBrowserEnv), floor at 10s.
   // The outer BrowserTaskScheduler.runHealthCheck() has its own 45s hard deadline.
@@ -733,14 +765,14 @@ export async function executeHealthCheck(
   }
 
   try {
-    return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT);
+    return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_PRIMARY_URL, signal);
   } catch (firstError: unknown) {
     // One retry: if the first attempt fails, the page may have been in a bad state
     // (challenge redirect, partial load, transient network blip). A second attempt
     // on a fresh page helps distinguish a real outage from a one-off failure.
     logToDebugFile('WARN', `[Worker-${workerId}] Health check attempt 1 failed: ${firstError instanceof Error ? firstError.message : String(firstError)}. Retrying once...`);
     try {
-      return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT);
+      return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_PRIMARY_URL, signal);
     } catch (retryError: unknown) {
       const retryMsg = retryError instanceof Error ? retryError.message : String(retryError);
       // A TIMEOUT is left to propagate: upstream (isBusyPoolHealthFailure) already
@@ -756,7 +788,7 @@ export async function executeHealthCheck(
         // healthy enough to research (search has its own retries/fallbacks downstream).
         logToDebugFile('WARN', `[Worker-${workerId}] Search-provider health probe failed twice (${retryMsg}); verifying open-web reachability via fallback endpoint...`);
         try {
-          const fallback = await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_FALLBACK_URL);
+          const fallback = await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_FALLBACK_URL, signal);
           logToDebugFile('WARN', `[Worker-${workerId}] Fallback endpoint reachable — open web is up, search provider is degraded. Proceeding with research.`);
           return fallback;
         } catch (fallbackError: unknown) {

package/src/infrastructure/browser/thread-worker.mjs

@@ -1060,6 +1060,17 @@ var init_logger = __esm({
           timestamp,
           level,
           ...getLogContext(),
+          // Every process writes to the SAME log file — up to three concurrent research
+          // runs plus the elected browser-pool leader — and until this field existed
+          // there was nothing to tell their lines apart. Reconstructing a lock
+          // contention from the log then meant inferring process boundaries from
+          // timing, which is guesswork: two runs a second apart interleave into what
+          // reads like one process contradicting itself. Twelve bytes a line buys
+          // attributable forensics.
+          //
+          // After the context spread, not before: which process wrote a line is not
+          // something a scoped context gets to claim otherwise.
+          pid: process.pid,
           message,
           ...firstError ? { errorMessage: redactSecrets(firstError.message), errorStack: redactSecrets(firstError.stack ?? "") } : {}
         };
@@ -1166,7 +1177,7 @@ var init_metrics = __esm({
       increment(name, value = 1, labels) {
         const key = this.getKey(name, labels);
         this.counters.set(key, (this.counters.get(key) ?? 0) + value);
-        if (name === "llm_cost_total" || name === "llm_tokens_total") {
+        if (name.startsWith("llm_")) {
           logger.debug(`[Metrics] Incremented ${name} by ${value}${labels ? ` (labels: ${JSON.stringify(labels)})` : ""}`);
         }
       }
@@ -1270,6 +1281,28 @@ var init_metrics = __esm({
       async measure(name, action, labels) {
         return this.getActive().measure(name, action, labels);
       }
+      /**
+       * Emit API that ALWAYS targets the session registry, ignoring any active run
+       * context.
+       *
+       * For process-lifetime callbacks that happen to be *registered* during a run.
+       * AsyncLocalStorage resolves at emit time from the async resource that fired,
+       * and for a listener installed on a long-lived emitter that resource is the
+       * emitter itself — created, for every lazily-initialized subsystem here, inside
+       * whichever run first needed it. So a worker-pool error handler installed during
+       * run 1 keeps writing into run 1's registry for the rest of the process: those
+       * counts disappear from every later run's summary (the registry was discarded
+       * when the run ended) and the dead registry stays reachable through the closure.
+       *
+       * Use this only where the emission is genuinely process-scoped. Anything that
+       * describes work a run asked for belongs in the run registry, which is what the
+       * plain emit methods above give you.
+       */
+      session = {
+        increment: (name, value = 1, labels) => this._session.increment(name, value, labels),
+        setGauge: (name, value, labels) => this._session.setGauge(name, value, labels),
+        observe: (name, value, labels) => this._session.observe(name, value, labels)
+      };
       // ── Session-level read API ─────────────────────────────────────────────
       /** Snapshot of the session registry (infrastructure / cross-run metrics). */
       getSessionSnapshot() {
@@ -2170,11 +2203,21 @@ async function extractSearchResults(page) {
     return found;
   });
 }
-async function executeSearchTask(_context, query) {
+function bindTaskAbortToPage(page, signal) {
+  if (!signal) return;
+  const onAbort = () => {
+    page.close().catch(() => {
+    });
+  };
+  if (signal.aborted) onAbort();
+  else signal.addEventListener("abort", onAbort, { once: true });
+}
+async function executeSearchTask(_context, query, signal) {
   const page = await createPageSafe(_context);
   const SEARCH_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SEARCH_TIMEOUT_MS"], 45e3);
   page.setDefaultTimeout(SEARCH_TIMEOUT);
   page.setDefaultNavigationTimeout(SEARCH_TIMEOUT);
+  bindTaskAbortToPage(page, signal);
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting search for: ${query}`);
     if (process.env["PI_RESEARCH_MOCK_SEARCH"] === "true") {
@@ -2387,10 +2430,11 @@ async function executeScrapeTask(_context, url, signal) {
 }
 var HEALTH_PRIMARY_URL = "https://lite.duckduckgo.com/lite/";
 var HEALTH_FALLBACK_URL = "https://example.com/";
-async function executeHealthCheckAttempt(_context, navTimeoutMs, url = HEALTH_PRIMARY_URL) {
+async function executeHealthCheckAttempt(_context, navTimeoutMs, url = HEALTH_PRIMARY_URL, signal) {
   const page = await createPageSafe(_context);
   page.setDefaultTimeout(navTimeoutMs);
   page.setDefaultNavigationTimeout(navTimeoutMs);
+  bindTaskAbortToPage(page, signal);
   try {
     const navStart = Date.now();
     await page.goto(url, { waitUntil: "domcontentloaded" });
@@ -2406,25 +2450,25 @@ async function executeHealthCheckAttempt(_context, navTimeoutMs, url = HEALTH_PR
     throw error;
   }
 }
-async function executeHealthCheck(_context, initMs) {
+async function executeHealthCheck(_context, initMs, signal) {
   const configuredMs = parseTimeoutMs(process.env["PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS"], 0);
   const HEALTH_TIMEOUT = configuredMs > 0 ? Math.max(1e4, configuredMs) : 1e4;
   if (initMs > 3e3) {
     logToDebugFile2("WARN", `[Worker-${workerId2}] Browser init was slow: ${initMs}ms \u2014 leaving less headroom for DDG navigation.`);
   }
   try {
-    return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT);
+    return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_PRIMARY_URL, signal);
   } catch (firstError) {
     logToDebugFile2("WARN", `[Worker-${workerId2}] Health check attempt 1 failed: ${firstError instanceof Error ? firstError.message : String(firstError)}. Retrying once...`);
     try {
-      return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT);
+      return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_PRIMARY_URL, signal);
     } catch (retryError) {
       const retryMsg = retryError instanceof Error ? retryError.message : String(retryError);
       const isTimeout = /tim(e|ed)\s*out|timeout/i.test(retryMsg);
       if (!isTimeout) {
         logToDebugFile2("WARN", `[Worker-${workerId2}] Search-provider health probe failed twice (${retryMsg}); verifying open-web reachability via fallback endpoint...`);
         try {
-          const fallback = await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_FALLBACK_URL);
+          const fallback = await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_FALLBACK_URL, signal);
           logToDebugFile2("WARN", `[Worker-${workerId2}] Fallback endpoint reachable \u2014 open web is up, search provider is degraded. Proceeding with research.`);
           return fallback;
         } catch (fallbackError) {
@@ -2528,7 +2572,7 @@ var ConfigSchema = Type.Object({
    *  never killed before it has used its full nav budget plus time waiting in the queue.
    *  (default: 10000) */
   BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2e3, maximum: 12e4, default: 1e4 }),
-  /** Timeout for coordinator/evaluator/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
+  /** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
    *  Not exposed in TUI — controlled via PI_RESEARCH_LLM_TIMEOUT_MS env var.
    *  Ceiling raised to 30min (matching RESEARCHER_TIMEOUT_MS): a slow model on a
    *  large-context synthesis/plan call can legitimately run >10min, and the old
@@ -2536,7 +2580,7 @@ var ConfigSchema = Type.Object({
    *  (e.g. 900000), causing those calls to time out mid-generation. */
   LLM_TIMEOUT_MS: Type.Number({ minimum: 6e4, maximum: 18e5, default: 3e5 }),
   /** Chain-of-thought "thinking" level for the engine's own LLM calls (coordinator,
-   *  evaluator, synthesis, JSON-repair, knowledge extraction) AND the researcher
+   *  router, synthesizer, JSON-repair, knowledge extraction) AND the researcher
    *  sub-agents. Default 'off': these calls emit structured JSON / cited reports, not
    *  open-ended reasoning, so thinking only burns the output-token budget (often
    *  truncating before the JSON/report block) and adds latency for at most a marginal
@@ -2551,22 +2595,26 @@ var ConfigSchema = Type.Object({
     Type.Literal("medium"),
     Type.Literal("high")
   ], { default: "off" }),
-  /** Max output tokens for the coordinator plan + mid-round evaluator decision (and,
-   *  when the evaluator synthesizes mid-round, the report). Default 16384. The blanket
+  /** Max output tokens for the coordinator's round-1 plan. Default 16384. The routing
+   *  decision has its OWN, much smaller ceiling (it emits a decision, never a report,
+   *  and cannot synthesize mid-round under the split protocol); the final report uses
+   *  SYNTHESIS_MAX_TOKENS below. The blanket
    *  4096 cap previously throttled every call and could be exhausted by a thinking block
    *  before any text was emitted. Capped at the model's real ceiling at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_PLANNING_MAX_TOKENS env var. */
   PLANNING_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
-  /** Max output tokens for the final synthesized research report (the forced-synthesis
-   *  evaluator call). Default 32768 so a full cited report is never truncated. Capped at
+  /** Max output tokens for the final synthesized research report (the synthesizer call,
+   *  and each pass of a reduce when the corpus does not fit in one). Default 32768 so a
+   *  full cited report is never truncated. It is also the reserve subtracted from the
+   *  model's context window when budgeting that corpus. Capped at
    *  the model's real maxTokens at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
   SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
   /** LLM Model override for researcher sub-agents and knowledge synthesis.
    *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
    *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
-   *  and the knowledge synthesis background LLM. The coordinator and evaluator always
-   *  use the caller's model (ctx.model).
+   *  and the knowledge synthesis background LLM. The coordinator and the research lead
+   *  always use the caller's model (ctx.model).
    */
   RESEARCH_MODEL: Type.Optional(Type.String()),
   /** Explicit directory for the knowledge store database (overrides default) */
@@ -2620,6 +2668,7 @@ function resolveHeadlessMode() {
   if (process.env["PI_RESEARCH_USE_XVFB"] === "true") return "virtual";
   return true;
 }
+var HEALTHCHECK_MIN_BUDGET_MS = 45e3 + 6e4;
 
 // src/infrastructure/browser/thread-worker-browser.ts
 var browser = null;
@@ -2734,11 +2783,24 @@ async function initBrowser() {
           });
           launchPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser launch rejection: ${err.message}`));
           let launchTimeoutId;
+          let raceLost = false;
+          launchPromise.then((b) => {
+            if (!raceLost) return;
+            logToDebugFile3("WARN", `[Worker-${workerId3}] Browser launch completed after its deadline; closing the orphaned instance.`);
+            try {
+              b?.close?.();
+            } catch {
+            }
+          }).catch(() => {
+          });
           try {
             return await Promise.race([
               launchPromise,
               new Promise((_, reject) => {
-                launchTimeoutId = setTimeout(() => reject(new Error(`Browser launch timed out after ${launchTimeoutMs}ms`)), launchTimeoutMs);
+                launchTimeoutId = setTimeout(() => {
+                  raceLost = true;
+                  reject(new Error(`Browser launch timed out after ${launchTimeoutMs}ms`));
+                }, launchTimeoutMs);
               })
             ]);
           } finally {
@@ -2763,10 +2825,23 @@ async function initBrowser() {
         const contextPromise = browser.newContext();
         contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
         let contextTimeoutId;
+        let contextRaceLost = false;
+        contextPromise.then((c) => {
+          if (!contextRaceLost) return;
+          logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
+          try {
+            c?.close?.();
+          } catch {
+          }
+        }).catch(() => {
+        });
         context = await Promise.race([
           contextPromise,
           new Promise((_, reject) => {
-            contextTimeoutId = setTimeout(() => reject(new Error("Browser context creation timed out after 30000ms")), 3e4);
+            contextTimeoutId = setTimeout(() => {
+              contextRaceLost = true;
+              reject(new Error("Browser context creation timed out after 30000ms"));
+            }, 3e4);
           })
         ]);
         if (contextTimeoutId) clearTimeout(contextTimeoutId);
@@ -2882,14 +2957,14 @@ async function runTask(data) {
     let result;
     if (type === "search") {
       if (!query) throw new Error("Search task requires a query");
-      const searchResult = await executeSearchTask(getContext(), query);
+      const searchResult = await executeSearchTask(getContext(), query, abortController.signal);
       result = { results: searchResult.results, duration: Date.now() - startTime, jitter: searchResult.jitter };
     } else if (type === "scrape") {
       if (!url) throw new Error("Scrape task requires a URL");
       const scrapeResult = await executeScrapeTask(getContext(), url, abortController.signal);
       result = { ...scrapeResult, duration: Date.now() - startTime };
     } else if (type === "healthcheck") {
-      const healthResult = await executeHealthCheck(getContext(), initMs);
+      const healthResult = await executeHealthCheck(getContext(), initMs, abortController.signal);
       result = { ...healthResult, duration: Date.now() - startTime };
     } else {
       result = { error: "Unknown task type", duration: Date.now() - startTime };

package/src/infrastructure/browser/thread-worker.ts

@@ -116,14 +116,14 @@ async function runTask(data: TaskData | undefined): Promise<TaskResult> {
     let result: any;
     if (type === 'search') {
       if (!query) throw new Error('Search task requires a query');
-      const searchResult = await executeSearchTask(getContext(), query);
+      const searchResult = await executeSearchTask(getContext(), query, abortController.signal);
       result = { results: searchResult.results, duration: Date.now() - startTime, jitter: searchResult.jitter };
     } else if (type === 'scrape') {
       if (!url) throw new Error('Scrape task requires a URL');
       const scrapeResult = await executeScrapeTask(getContext(), url, abortController.signal);
       result = { ...scrapeResult, duration: Date.now() - startTime };
     } else if (type === 'healthcheck') {
-      const healthResult = await executeHealthCheck(getContext(), initMs);
+      const healthResult = await executeHealthCheck(getContext(), initMs, abortController.signal);
       result = { ...healthResult, duration: Date.now() - startTime };
     } else {
       result = { error: 'Unknown task type', duration: Date.now() - startTime };

package/src/infrastructure/browser/worker-pool-manager.ts

@@ -28,6 +28,55 @@ const __dirname = dirname(__filename);
 /**
  * Worker pool manager for browser operations.
  */
+/**
+ * The pool's dispatch shape: how a worker is chosen, and how much each may hold.
+ *
+ * Separated out so it is testable. The claim this configuration makes is behavioural
+ * — that no task is parked behind a busy worker while another idles — and the first
+ * test written for it asserted that this file contained the string
+ * "WorkerChoiceStrategies.LEAST_USED", which passes against an implementation that
+ * dispatches nothing.
+ *
+ * LEAST_USED picks the node minimising executing + queued. ROUND_ROBIN is a bare
+ * modular cursor whose only filter is a one-time `ready` flag — it cannot see load,
+ * and poolifier consults per-node capacity only AFTER the strategy has chosen. A task
+ * handed to a busy node is queued ON that node and never re-routed, so one degraded
+ * worker takes a share of every burst with it: on 2026-08-15 a worker whose browser
+ * launch hung was still handed roughly one turn in six, and the run lost exactly the
+ * three queries that landed on it while five workers sat idle.
+ *
+ * The guarantee is conditional, not absolute, and worth stating precisely: a
+ * load-reading strategy never parks a task WHILE A FREE NODE EXISTS. The
+ * PriorityTaskQueue above normally makes sure one does, by capping in-flight work at
+ * exactly WORKER_THREADS x WORKER_CONCURRENCY — which is why a WORKER_CONCURRENCY
+ * change has to rebuild the pool, or the two layers disagree about capacity. But that
+ * cap is released early when a caller aborts a task still running on a worker, and a
+ * respawning worker is not `ready`, so there are windows where every node is genuinely
+ * full. In those, ties resolve to the lowest-index node and a task does park.
+ *
+ * Which is also why poolifier's stolen-task accounting defect is made rare rather than
+ * unreachable: stealing fires only when something is queued on a node, and a stolen
+ * task's completion decrements the ORIGINAL node's executing count instead of the one
+ * that ran it, freeing a slot on a worker that is still working. That produced 1,425
+ * genuinely overlapping task pairs across 37 days at a configured concurrency of 1.
+ *
+ * LEAST_BUSY was the other candidate and is worse here: it minimises accumulated
+ * run+wait time, which turns on per-task performance measurement for every task and
+ * still lets a busy fast worker outrank an idle slow one. LEAST_USED overrides no
+ * statistics requirements, so it costs nothing extra.
+ */
+export function buildWorkerPoolOptions(config: Config): {
+    workerChoiceStrategy: typeof WorkerChoiceStrategies[keyof typeof WorkerChoiceStrategies];
+    enableTasksQueue: boolean;
+    tasksQueueOptions: { concurrency: number };
+} {
+    return {
+        workerChoiceStrategy: WorkerChoiceStrategies.LEAST_USED,
+        enableTasksQueue: true,
+        tasksQueueOptions: { concurrency: config.WORKER_CONCURRENCY },
+    };
+}
+
 export class WorkerPoolManager implements IService {
     readonly name = ServiceNames.WORKER_POOL_MANAGER;
     lifecycle = ServiceLifecycle.UNINITIALIZED;
@@ -35,6 +84,12 @@ export class WorkerPoolManager implements IService {
     private pool: any | null = null;
     private poolInitializationPromise: Promise<any> | null = null;
     private currentWorkerCount: number | null = null;
+    /** Per-worker task concurrency the LIVE pool was built with. Baked into
+     *  poolifier's tasksQueueOptions at construction and not changeable after, so a
+     *  change has to recreate the pool — the upper PriorityTaskQueue recomputes its
+     *  own capacity from the same two values on every task, and the two disagreeing
+     *  is what lets a task be queued on a worker that the queue believes is free. */
+    private currentWorkerConcurrency: number | null = null;
     private consecutiveErrors: number = 0;
     private isShuttingDown: boolean = false;
     // Monotonic shutdown generation. Each shutdown() bumps it, so an in-flight
@@ -95,7 +150,8 @@ export class WorkerPoolManager implements IService {
 
         // If pool exists, worker count matches, and we are not shutting down,
         // return the existing pool immediately.
-        if (this.pool && this.currentWorkerCount === maxWorkers) {
+        const workerConcurrency = (config || getConfig()).WORKER_CONCURRENCY;
+        if (this.pool && this.currentWorkerCount === maxWorkers && this.currentWorkerConcurrency === workerConcurrency) {
             return this.pool;
         }
 
@@ -110,14 +166,15 @@ export class WorkerPoolManager implements IService {
         const myGeneration = this.generation;
         this.poolInitializationPromise = (async () => {
             try {
-                if (this.pool && this.currentWorkerCount !== maxWorkers) {
-                    logger.log(`[WorkerPoolManager] Worker count changed from ${this.currentWorkerCount} to ${maxWorkers}, recreating pool...`);
+                if (this.pool && (this.currentWorkerCount !== maxWorkers || this.currentWorkerConcurrency !== workerConcurrency)) {
+                    logger.log(`[WorkerPoolManager] Pool shape changed (workers ${this.currentWorkerCount} -> ${maxWorkers}, concurrency ${this.currentWorkerConcurrency} -> ${workerConcurrency}), recreating pool...`);
                     this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
                     await this.pool.destroy();
                     this.pool = null;
                 }
 
                 this.currentWorkerCount = maxWorkers;
+                this.currentWorkerConcurrency = workerConcurrency;
 
                 // Install the master-side IPC guard BEFORE any worker is forked, so a
                 // `write EPIPE` on a closing worker channel (poolifier dispatch racing a
@@ -161,7 +218,6 @@ export class WorkerPoolManager implements IService {
                 void cleanupOrphanedCamoufoxProcesses()
                     .catch((e) => logger.debug('[WorkerPoolManager] Startup orphan sweep failed:', e));
 
-                const workerConcurrency = (config || getConfig()).WORKER_CONCURRENCY;
 
                 // thread-worker.mjs is the esbuild-compiled bundle of thread-worker.ts and its
                 // local imports. Using a pre-compiled JS file eliminates any TypeScript loading
@@ -205,10 +261,15 @@ export class WorkerPoolManager implements IService {
                             return;
                         }
                         this.consecutiveErrors++;
-                        metrics.increment('browser_pool_errors_total', 1);
+                        // Session-scoped on purpose. This handler is installed on the pool,
+                        // and the pool is built lazily inside whichever research run first
+                        // needed a browser — so the run-registry context every later
+                        // invocation inherits is that FIRST run's, long after it was
+                        // discarded. Pool health is process state, not run accounting.
+                        metrics.session.increment('browser_pool_errors_total', 1);
                         logger.error('[WorkerPoolManager] Cluster Error:', e);
                         if (this.consecutiveErrors >= 3) {
-                            metrics.increment('browser_pool_unhealthy_events_total', 1);
+                            metrics.session.increment('browser_pool_unhealthy_events_total', 1);
                             logger.error(`[WorkerPoolManager] Worker pool may be unhealthy: ${this.consecutiveErrors} consecutive errors. Consider restarting.`);
                             if (this.onPoolError) {
                                 this.onPoolError(e, this.consecutiveErrors);
@@ -235,11 +296,7 @@ export class WorkerPoolManager implements IService {
                             }
                         }
                     },
-                    workerChoiceStrategy: WorkerChoiceStrategies.ROUND_ROBIN,
-                    enableTasksQueue: true,
-                    tasksQueueOptions: {
-                        concurrency: workerConcurrency, // configurable via PI_RESEARCH_WORKER_CONCURRENCY
-                    }
+                    ...buildWorkerPoolOptions(config || getConfig()),
                 });
 
                 // Secondary race check: if a shutdown() was called concurrent with
@@ -315,6 +372,60 @@ export class WorkerPoolManager implements IService {
         if (this.consecutiveErrors > 0) this.consecutiveErrors--;
     }
 
+    /**
+     * Record how the pool is actually distributing work.
+     *
+     * The head-of-line failure this pool used to have was invisible to every metric
+     * and every log line: tasks parked on one node while other nodes idled, and the
+     * only trace was a PriorityTaskQueue reporting free slots and an empty backlog
+     * while queries timed out. poolifier computes all of it on `pool.info` and
+     * nothing read it.
+     *
+     * `browser_pool_parked_tasks` is the number that should stay at zero: a task
+     * queued on a worker node while some other node is idle. Non-zero means the
+     * choice strategy handed work to a busy node when a free one existed — which is
+     * also the precondition for poolifier's task stealing, and therefore for its
+     * stolen-task accounting defect. `stolenTasks` is recorded for the same reason.
+     *
+     * Two limits, stated because a gauge that reads zero is otherwise taken as proof:
+     * `idleWorkerNodes` counts nodes that are READY and free, so during a respawn
+     * window a genuinely parked task is attributed to legitimate saturation; and with
+     * every node busy the figure is zero by construction, which is correct but is not
+     * evidence the strategy is choosing well. `browser_pool_idle_workers` is exported
+     * alongside so the denominator is visible rather than inferred.
+     *
+     * With no pool the gauges are ZEROED rather than left alone. Returning early held
+     * the last live reading forever — so a burst that ended with tasks parked left a
+     * standing non-zero gauge for a pool that no longer exists, and every later read
+     * of it was a report about a dead process.
+     *
+     * Session-scoped, for the same reason as the pool's error counters above. The only
+     * caller is the leader's 5s leadership tick — a self-rescheduling timer chain
+     * started when this process won the election, i.e. inside whichever research run
+     * first needed a browser. AsyncLocalStorage resolves at emit time from the async
+     * resource that fired, so a plain `metrics.setGauge` here would write into that
+     * FIRST run's registry for the life of the process: invisible to every later run's
+     * summary, and holding a discarded registry reachable through the timer. The pool's
+     * dispatch shape is process state, not run accounting.
+     */
+    recordDispatchHealth(): void {
+        const info = (this.pool as unknown as { info?: Record<string, number> } | null)?.info;
+        if (!info) {
+            metrics.session.setGauge('browser_pool_parked_tasks', 0);
+            metrics.session.setGauge('browser_pool_queued_tasks', 0);
+            metrics.session.setGauge('browser_pool_idle_workers', 0);
+            return;
+        }
+        const queued = info['queuedTasks'] ?? 0;
+        const idle = info['idleWorkerNodes'] ?? 0;
+        metrics.session.setGauge('browser_pool_parked_tasks', idle > 0 ? queued : 0);
+        metrics.session.setGauge('browser_pool_queued_tasks', queued);
+        metrics.session.setGauge('browser_pool_idle_workers', idle);
+        // Cumulative, not a level: never zeroed on teardown, or a restart would read
+        // as the defect having been fixed.
+        metrics.session.setGauge('browser_pool_stolen_tasks', info['stolenTasks'] ?? 0);
+    }
+
     /**
      * Schedule an out-of-band pool reset. Called from poolifier event handlers
      * where destroying the pool synchronously would deadlock. Waits 1 s so the
@@ -334,12 +445,15 @@ export class WorkerPoolManager implements IService {
         // is still being destroyed. Instead, use a flag to signal reset is in progress.
         this._resetInProgress = true;
         this.currentWorkerCount = null;
+        this.currentWorkerConcurrency = null;
         this.consecutiveErrors = 0;
         // Pin the generation this reset belongs to. shutdown() bumps generation, so if a
         // shutdown (and a subsequent re-init) happens while this timer is pending, the
         // generation no longer matches and the timer must NOT touch the new pool's state.
         const myGen = this.generation;
-        metrics.increment('browser_pool_auto_recoveries_total', 1);
+        // Session-scoped: every caller of schedulePoolReset is one of the pool's own
+        // lifecycle handlers, which carry the async context of pool construction.
+        metrics.session.increment('browser_pool_auto_recoveries_total', 1);
         logger.info('[WorkerPoolManager] Pool scheduled for auto-recovery; next ensurePool() will wait for old pool destruction.');
         // Destroy the old pool asynchronously after the event handler returns.
         const t = setTimeout(async () => {
@@ -427,6 +541,7 @@ export class WorkerPoolManager implements IService {
 
         this.poolInitializationPromise = null;
         this.currentWorkerCount = null;
+        this.currentWorkerConcurrency = null;
         this.consecutiveErrors = 0;
         // Clear any in-flight auto-recovery flag too: if a schedulePoolReset() timer
         // was pending when shutdown ran, leaving this true makes the next ensurePool()

package/src/infrastructure/embedding/embedding-client.ts

@@ -72,8 +72,19 @@ export class EmbeddingClient implements IEmbedder {
 
   // ---- Health ----
 
-  async fetchHealth(): Promise<void> {
-    const result = await this.getRequest<{ status: string; device: string; dimension: number | null }>('/health');
+  /**
+   * `timeoutMs` defaults to the general-purpose 120s used for real embed requests, but the
+   * embedding-leader poll loop (embedding-factory.ts) needs a much shorter deadline: it uses
+   * a failed health check as one "unreachable poll" toward declaring a leader's registration
+   * stale, on the documented assumption that each poll costs "the poll interval plus up to
+   * the [probe] timeout" (~2.5s). At the 120s default that assumption is off by 48x — a
+   * leader whose event loop is merely busy (not dead) can take the full 120s to answer, so
+   * the poll loop's outer wait times out and retries long before its own unreachable-poll
+   * counter would ever fire. Callers that need the fast-fail poll semantics must pass an
+   * explicit short timeout.
+   */
+  async fetchHealth(timeoutMs?: number): Promise<void> {
+    const result = await this.getRequest<{ status: string; device: string; dimension: number | null }>('/health', timeoutMs);
     this._device = result.device;
     if (result.dimension !== null) {
       this.dimension = result.dimension;
@@ -82,9 +93,8 @@ export class EmbeddingClient implements IEmbedder {
 
   // ---- Internal HTTP helpers ----
 
-  private getRequest<T>(path: string): Promise<T> {
+  private getRequest<T>(path: string, timeoutMs = 120_000): Promise<T> {
     return new Promise((resolve, reject) => {
-      const timeoutMs = 120_000;
       let resolved = false;
       let req: http.ClientRequest | undefined;
       const timer = setTimeout(() => {