@lincoln504/pi-research 1.4.1 → 1.4.3

package/src/infrastructure/browser/browser-task-scheduler.ts

@@ -23,7 +23,7 @@ import type { IScheduler, TaskDispatchListener } from '../../core/interfaces/sch
 import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowserProcesses } from './browser-cleanup.ts';
 import { raceWithDeadline } from '../../utils/safe-unref.ts';
 import { PriorityTaskQueue } from './priority-task-queue.ts';
-import { getHealthCheckBudgetMs } from './config.ts';
+import { getHealthCheckBudgetMs, COLD_START_ALLOWANCE_MS } from './config.ts';
 
 /**
  * Grace added to the worker's own task budget before the dead-worker backstop
@@ -273,12 +273,17 @@ export class BrowserTaskScheduler implements IScheduler {
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
         const startTime = Date.now();
 
-        // Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + a queue-wait/overhead
-        // margin (BROWSER_TASK_TIMEOUT_MS). Deriving from SEARCH_TIMEOUT_MS keeps the task
-        // ceiling coherent with the worker's own nav timeout — using a flat base here would
-        // kill a search before it had used its full nav budget.
+        // Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + an overhead margin
+        // (BROWSER_TASK_TIMEOUT_MS) + a cold-start allowance (COLD_START_ALLOWANCE_MS).
+        // Deriving from SEARCH_TIMEOUT_MS keeps the task ceiling coherent with the
+        // worker's own nav timeout — using a flat base here would kill a search before
+        // it had used its full nav budget. The cold-start term matters because eager
+        // browser warmup is deliberately disabled: the first search dispatched to a
+        // freshly created or just-reset worker pays a real Camoufox launch + context
+        // creation inline, before any navigation even starts — without this term that
+        // legitimate cold start reads as a dead/wedged worker (see COLD_START_ALLOWANCE_MS).
         const cfg = config || getConfig();
-        const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
+        const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS;
 
         // The deadline measures EXECUTION, not queue wait: the timer is armed by
         // `startDeadline()` from inside the enqueued callback, i.e. once the queue has
@@ -413,11 +418,13 @@ export class BrowserTaskScheduler implements IScheduler {
         this.resetIdleTimer();
         const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
         const startTime = Date.now();
-        // Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + the queue-wait/overhead
-        // margin (BROWSER_TASK_TIMEOUT_MS), mirroring the search path so both stay coherent.
+        // Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + an overhead margin,
+        // mirroring the search path so both stay coherent. The mocked branch skips the
+        // cold-start allowance deliberately: PI_RESEARCH_MOCK_SCRAPE implies no real
+        // Camoufox launch is expected for scrape tasks, so there is nothing to budget for.
         const cfg = config || getConfig();
         const isMocking = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
-        const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS);
+        const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS);
 
         // Armed on dispatch, not on enqueue — see runSearch for why.
         // See runSearch: dispatch time is tracked for measurement, not for the deadline.

package/src/infrastructure/browser/config.ts

@@ -346,13 +346,50 @@ function isFullMockMode(): boolean {
  *     and a budget below that sum makes the fallback ladder unreachable — so raising
  *     HEALTH_CHECK_TIMEOUT_MS used to make the health check strictly worse.
  *
- * Floored at the historical 105s so default installs are unchanged.
+ * Floored at the historical 105s, kept as a floor rather than a default: the
+ * derived value now exceeds it even at default timeouts once COLD_START_ALLOWANCE_MS
+ * (below) is folded into `longestSlotHold` — a cold Firefox launch was always able to
+ * hold a slot for close to that long, the 105s floor just didn't know it. The floor
+ * still matters for a hypothetical config where the derived value would otherwise land
+ * below it.
  */
 export const HEALTHCHECK_MIN_BUDGET_MS = 45_000 + 60_000;
 
 /** Navigation attempts executeHealthCheck makes: primary, retry, fallback endpoint. */
 const HEALTHCHECK_NAV_ATTEMPTS = 3;
 
+/**
+ * Hard ceiling on a single Camoufox browser launch (see launchOnce() in
+ * thread-worker-browser.ts, the single source of truth this mirrors — CI runners
+ * (2 vCPU) can take up to ~60s to start Firefox under load, floored generously
+ * above that).
+ */
+export const BROWSER_LAUNCH_TIMEOUT_MS = 90_000;
+
+/**
+ * Hard ceiling on creating one BrowserContext against an already-launched browser
+ * (see acquireTaskContext() in thread-worker-browser.ts, the single source of
+ * truth this mirrors).
+ */
+export const CONTEXT_CREATION_TIMEOUT_MS = 30_000;
+
+/**
+ * Worst-case time a worker can spend getting a browser+context ready before it
+ * starts a task's actual work. Eager warmup is deliberately disabled (thundering-
+ * herd avoidance — see thread-worker.ts), so the first task dispatched to a
+ * freshly created or just-reset worker pays this in full, inline with the task
+ * itself. Every deadline that bounds how long a task may legitimately hold a
+ * worker slot has to include this on top of its own nav budget, or a completely
+ * normal cold start on a fresh pool reads as a dead/wedged worker — the false
+ * positive `runSearch`/`runScrape`'s task-timeout ceiling and this budget were
+ * both built to eliminate, just triggered by a different clock.
+ *
+ * Deliberately NOT part of BROWSER_TASK_TIMEOUT_MS: that value is user-tunable
+ * overhead for per-task work, and cold-start cost isn't something a user should
+ * need to account for by hand when adjusting it.
+ */
+export const COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
+
 export function getHealthCheckBudgetMs(config?: Config): number {
     const c = (config || getConfig()) as Partial<Config>;
     // Each field is read defensively. A missing or non-finite value would otherwise
@@ -363,7 +400,8 @@ export function getHealthCheckBudgetMs(config?: Config): number {
         typeof value === 'number' && Number.isFinite(value) ? value : fallback;
     const longestSlotHold =
         Math.max(ms(c.SEARCH_TIMEOUT_MS, 45_000), ms(c.SCRAPE_TIMEOUT_MS, 15_000))
-        + ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000);
+        + ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000)
+        + COLD_START_ALLOWANCE_MS;
     // Mirrors the worker's own floor in executeHealthCheck.
     const probeWork = HEALTHCHECK_NAV_ATTEMPTS * Math.max(10_000, ms(c.HEALTH_CHECK_TIMEOUT_MS, 10_000));
     return Math.max(HEALTHCHECK_MIN_BUDGET_MS, longestSlotHold + probeWork);

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -11,21 +11,23 @@
 import { platform } from 'node:os';
 import { setupMocking } from './thread-worker-messaging.ts';
 import { redactSecrets } from '../../utils/log-utils.ts';
-import { resolveHeadlessMode } from './config.ts';
+import { resolveHeadlessMode, BROWSER_LAUNCH_TIMEOUT_MS, CONTEXT_CREATION_TIMEOUT_MS } from './config.ts';
 
 let browser: any = null;
-let context: any = null;
 let initPromise: Promise<void> | null = null;
 let workerId: string = '';
 
 // How many tasks this worker process currently has in flight. poolifier's
 // tasksQueueOptions.concurrency (WORKER_CONCURRENCY, default 2 — see
 // worker-pool-manager.ts) dispatches more than one task at a time to the SAME
-// worker process, and every dispatched task shares this module's browser/
-// context singletons (each gets its own page via createPageSafe(context), but
-// the context/browser themselves are one shared instance). resetBrowser()
-// uses this to avoid tearing that shared instance down out from under a
-// sibling task that is still actively using it — see resetBrowser() below.
+// worker process, and every dispatched task shares this module's browser
+// singleton (each gets its own FRESH BrowserContext via acquireTaskContext()
+// below — a real Firefox process is too expensive to launch per task, but a
+// context is cheap, and per-task contexts keep cookies/storage/route-mocking
+// from one task bleeding into a concurrent or later, unrelated one).
+// resetBrowser() uses this to avoid tearing the shared BROWSER down out from
+// under a sibling task that is still actively using it — see resetBrowser()
+// below.
 let activeTaskCount = 0;
 // A reset that resetBrowser() deferred because sibling tasks were still
 // active, to be applied once the last one finishes.
@@ -119,12 +121,12 @@ function logToDebugFile(level: string, ...args: any[]): void {
  * Initialize the browser instance
  */
 export async function initBrowser(): Promise<void> {
-  if (isBrowserConnected() && context) return;
+  if (isBrowserConnected()) return;
   if (initPromise) return initPromise;
 
   initPromise = (async () => {
     try {
-      if (!isBrowserConnected() || !context) {
+      if (!isBrowserConnected()) {
         logToDebugFile('INFO', `[Worker-${workerId}] Initializing browser instance...`);
 
         // ---------------------------------------------------------------------
@@ -196,8 +198,12 @@ export async function initBrowser(): Promise<void> {
         //
         // 90s launch timeout: CI runners (2 vCPU) need up to 60s to start Firefox
         // when it has both CPUs. The old 45s limit was too tight and caused silent
-        // init failures under resource pressure.
-        const launchTimeoutMs = 90000;
+        // init failures under resource pressure. Imported from config.ts (single
+        // source of truth) rather than a local literal — the scheduler's task
+        // deadlines (browser-task-scheduler.ts) and the healthcheck budget
+        // (getHealthCheckBudgetMs) both have to budget for this exact worst case,
+        // via COLD_START_ALLOWANCE_MS.
+        const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
         const launchOnce = async (headless: boolean | 'virtual') => {
           const launchPromise = Camoufox({
             headless,
@@ -295,35 +301,6 @@ export async function initBrowser(): Promise<void> {
         }
 
         browser = launchedBrowser;
-        // newContext() can hang if the browser process becomes unresponsive immediately
-        // after launch (e.g. OOM, GPU crash). Guard with a hard timeout.
-        const contextPromise = browser.newContext();
-        contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
-        let contextTimeoutId: NodeJS.Timeout | undefined;
-        let contextRaceLost = false;
-        // Same adoption as the launch above. Here the catch below CAN close the
-        // browser (it is assigned by now), which disposes the stray context with it —
-        // but only when initBrowser is entered with a dead browser, and it leaves the
-        // context open for the window in between. Closing it directly is unconditional.
-        contextPromise.then((c: any) => {
-          if (!contextRaceLost) return;
-          logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
-          try { c?.close?.(); } catch { /* already gone */ }
-        }).catch(() => { /* rejection already logged above */ });
-        context = await Promise.race([
-          contextPromise,
-          new Promise<never>((_, reject) => {
-            contextTimeoutId = setTimeout(() => {
-              contextRaceLost = true;
-              reject(new Error('Browser context creation timed out after 30000ms'));
-            }, 30000);
-          })
-        ]);
-        if (contextTimeoutId) clearTimeout(contextTimeoutId);
-        
-        // Setup mocking for CI if enabled
-        await setupMocking(context);
-        
         logToDebugFile('INFO', `[Worker-${workerId}] Browser initialized.`);
       }
     } catch (e: unknown) {
@@ -334,7 +311,6 @@ export async function initBrowser(): Promise<void> {
         Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during failed init: ${err.message}`));
       }
       browser = null;
-      context = null;
       const msg = e instanceof Error ? e.message : String(e);
 
       if (msg.includes('Camoufox is not installed') || msg.includes('Version information not found')) {
@@ -361,16 +337,60 @@ export async function initBrowser(): Promise<void> {
 }
 
 /**
- * Get the current browser context
+ * Create a FRESH BrowserContext for one task, against the shared browser
+ * process. Call once per task, right after initBrowser() resolves; the
+ * caller owns the returned context and must close it itself (in a `finally`,
+ * covering the success, error, AND timeout-abort paths) once the task ends.
+ *
+ * Never reused across tasks — that was the bug this replaced. Two tasks
+ * dispatched concurrently to the same worker (WORKER_CONCURRENCY > 1, the
+ * default) used to share one BrowserContext, so cookies/storage a site set
+ * for one task's request were visible to a different, unrelated task's later
+ * request to the same site — a content-consistency gap with no counterpart
+ * anywhere else in this codebase, which scopes browser-adjacent state
+ * (circuit breakers, scrape caches) per session deliberately.
+ *
+ * The launch race/timeout/orphan-adoption pattern below is the exact one
+ * initBrowser() used when context creation lived there (moved here wholesale,
+ * not reinvented): newContext() can hang if the browser process is
+ * unresponsive (OOM, GPU crash), so it is guarded with a hard timeout, and a
+ * context that finishes creating AFTER that timeout is adopted and closed
+ * rather than left to leak.
  */
-export function getContext(): any {
-  return context;
+export async function acquireTaskContext(): Promise<any> {
+  const contextPromise = browser.newContext();
+  contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
+  let contextTimeoutId: NodeJS.Timeout | undefined;
+  let contextRaceLost = false;
+  contextPromise.then((c: any) => {
+    if (!contextRaceLost) return;
+    logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
+    try { c?.close?.(); } catch { /* already gone */ }
+  }).catch(() => { /* rejection already logged above */ });
+  let newContext: any;
+  try {
+    newContext = await Promise.race([
+      contextPromise,
+      new Promise<never>((_, reject) => {
+        contextTimeoutId = setTimeout(() => {
+          contextRaceLost = true;
+          reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
+        }, CONTEXT_CREATION_TIMEOUT_MS);
+      })
+    ]);
+  } finally {
+    if (contextTimeoutId) clearTimeout(contextTimeoutId);
+  }
+
+  // Route interception is per-context, so mocking must be (re-)applied to
+  // every fresh context, not just the first one ever created.
+  await setupMocking(newContext);
+
+  return newContext;
 }
 
 function doResetBrowser(): void {
-  if (context) Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during reset: ${err.message}`));
   if (browser) Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during reset: ${err.message}`));
-  context = null;
   browser = null;
 }
 
@@ -407,7 +427,9 @@ export async function cleanupBrowser(): Promise<void> {
   const timeoutMs = 2000;
   
   const cleanup = async () => {
-    if (context) await Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during cleanup: ${err.message}`));
+    // Closes every context still open under it (any task whose own close()
+    // hasn't run yet) as a side effect — no separate context bookkeeping
+    // needed here now that contexts are task-owned, not module-owned.
     if (browser) await Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during cleanup: ${err.message}`));
   };
 
@@ -419,7 +441,6 @@ export async function cleanupBrowser(): Promise<void> {
   } catch {
     // Ignore timeout — we just want to ensure cleanup doesn't hang indefinitely
   } finally {
-    context = null;
     browser = null;
   }
 }

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -433,6 +433,13 @@ export async function executeScrapeTask(
   let poisonedError: Error | null = null;
   const pendingAddrChecks: Array<Promise<void>> = [];
 
+  // Set by the response-side early-abort check (see page.on('response') below) when the
+  // main-frame document's OWN headers already declare a PDF larger than MAX_PDF_SIZE.
+  // Declared outside the try for the same reason as poisonedError: the catch below needs
+  // to surface this instead of the secondary 'Target closed' error the triggering
+  // page.close() produces.
+  let oversizedPdfError: Error | null = null;
+
   try {
     logToDebugFile('DEBUG', `[Worker-${workerId}] Starting scrape for: ${url}`);
 
@@ -511,6 +518,42 @@ export async function executeScrapeTask(
       try {
         if (resp.request().resourceType() === 'document' && resp.frame() === page.mainFrame()) {
           finalMainFrameAddr = resp.serverAddr().catch(() => null);
+
+          // Early-abort a declared-oversized PDF as soon as its headers arrive, rather
+          // than waiting for page.goto() to finish downloading the whole thing first.
+          // 'response' fires on headers-received, well before the body is fully
+          // transferred — for a direct navigation to a PDF URL, Playwright's goto()
+          // does not resolve until the download completes, so without this the full
+          // (possibly multi-hundred-MB) body is pulled into the browser process before
+          // the PDF branch below ever gets to check Content-Length.
+          //
+          // This closes the gap only for a server that HONESTLY declares an oversized
+          // Content-Length up front. A chunked-transfer response (no Content-Length) or
+          // one that understates its length and then sends far more cannot be caught
+          // this way — Playwright's public API exposes no incremental
+          // bytes-received/streaming hook to abort mid-body on either engine. That
+          // residual case is bounded instead by the task's own SCRAPE_TIMEOUT_MS +
+          // BROWSER_TASK_TIMEOUT_MS deadline (see the onAbort/page.close() wiring
+          // above), which forcibly closes the page — and with it any in-flight
+          // download — once the task's time budget is exhausted, converting an
+          // otherwise-unbounded read into one bounded by (bandwidth x that budget).
+          const check2: Promise<void> = Promise.resolve(resp.headerValue('content-type'))
+            .then(async (ct: string | null) => {
+              // Matches the PDF branch below AND the fetch layer's own detection
+              // (web-scraper.ts): a server can mislabel or omit content-type, so a
+              // `.pdf` URL extension is an equally-trusted signal, not just the header.
+              const looksLikePdf = ct?.includes('application/pdf') || url.toLowerCase().endsWith('.pdf');
+              if (!looksLikePdf || oversizedPdfError || poisonedError) return;
+              const declared = parseInt((await resp.headerValue('content-length').catch(() => null)) || '', 10);
+              if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+                const sizeMB = Math.round(declared / 1024 / 1024);
+                oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+                logToDebugFile('DEBUG', `[Worker-${workerId}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
+                page.close().catch(() => {});
+              }
+            })
+            .catch(() => {});
+          pendingAddrChecks.push(check2);
         }
         const check: Promise<void> = Promise.resolve(resp.serverAddr())
           .then((addr: { ipAddress?: string } | null) => {
@@ -562,7 +605,14 @@ export async function executeScrapeTask(
     // 403/503 — the HTML branch defers to the challenge wait below for that.
     const status = response?.status?.() ?? 0;
 
-    if (contentType.includes('application/pdf')) {
+    // `.pdf` extension is an equally-trusted signal alongside the header (matches
+    // the fetch layer's own detection in web-scraper.ts): a server that mislabels
+    // or omits content-type would otherwise fall into the HTML branch below, whose
+    // page.content() would serialize Firefox's PDF-viewer chrome instead of
+    // extracting the document — a silent content loss for exactly the URLs this
+    // feature advertises as supported, and specifically for the malformed/oddly-
+    // served PDFs most likely to reach this browser fallback in the first place.
+    if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
       // An errored "PDF" is an error page: fail BEFORE buffering its body.
       if (status >= 400) throw new Error(`HTTP ${status}`);
@@ -576,6 +626,10 @@ export async function executeScrapeTask(
         const sizeMB = Math.round(declaredLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
+      // The response-side early-abort check above (page.on('response')) may already have
+      // decided this and be in the middle of closing the page — skip the (redundant,
+      // about-to-fail) download attempt rather than race it.
+      if (oversizedPdfError) throw oversizedPdfError;
       const buffer = await response.body();
       // ...then the actual bytes: the header is advisory and absent on chunked
       // responses, so the byte count is the check that actually bounds memory.
@@ -692,9 +746,12 @@ export async function executeScrapeTask(
     return { contentType, html, jitter };
   } catch (error) {
     await page.close().catch((err: any) => logToDebugFile('DEBUG', `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
-    // A poison-triggered page.close() makes the in-flight goto/content() reject
-    // with 'Target closed'; report the SSRF poisoning (a benign-classified
-    // 'Fetch blocked:' failure), not the secondary teardown error.
+    // An early-abort-triggered page.close() (oversized PDF, or SSRF poisoning) makes
+    // the in-flight goto/content()/body() reject with 'Target closed'; report the
+    // actual reason (a benign-classified failure) instead of that secondary teardown
+    // error. oversizedPdfError takes priority since it can only be set on the PDF path,
+    // where poisonedError (an HTML-only DOM-exfil concern) is not meaningful.
+    if (oversizedPdfError) throw oversizedPdfError;
     if (poisonedError) throw poisonedError;
     throw error;
   } finally {