@lincoln504/pi-research 1.3.12 → 1.4.3

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -11,21 +11,23 @@
 import { platform } from 'node:os';
 import { setupMocking } from './thread-worker-messaging.ts';
 import { redactSecrets } from '../../utils/log-utils.ts';
-import { resolveHeadlessMode } from './config.ts';
+import { resolveHeadlessMode, BROWSER_LAUNCH_TIMEOUT_MS, CONTEXT_CREATION_TIMEOUT_MS } from './config.ts';
 
 let browser: any = null;
-let context: any = null;
 let initPromise: Promise<void> | null = null;
 let workerId: string = '';
 
 // How many tasks this worker process currently has in flight. poolifier's
 // tasksQueueOptions.concurrency (WORKER_CONCURRENCY, default 2 — see
 // worker-pool-manager.ts) dispatches more than one task at a time to the SAME
-// worker process, and every dispatched task shares this module's browser/
-// context singletons (each gets its own page via createPageSafe(context), but
-// the context/browser themselves are one shared instance). resetBrowser()
-// uses this to avoid tearing that shared instance down out from under a
-// sibling task that is still actively using it — see resetBrowser() below.
+// worker process, and every dispatched task shares this module's browser
+// singleton (each gets its own FRESH BrowserContext via acquireTaskContext()
+// below — a real Firefox process is too expensive to launch per task, but a
+// context is cheap, and per-task contexts keep cookies/storage/route-mocking
+// from one task bleeding into a concurrent or later, unrelated one).
+// resetBrowser() uses this to avoid tearing the shared BROWSER down out from
+// under a sibling task that is still actively using it — see resetBrowser()
+// below.
 let activeTaskCount = 0;
 // A reset that resetBrowser() deferred because sibling tasks were still
 // active, to be applied once the last one finishes.
@@ -119,12 +121,12 @@ function logToDebugFile(level: string, ...args: any[]): void {
  * Initialize the browser instance
  */
 export async function initBrowser(): Promise<void> {
-  if (isBrowserConnected() && context) return;
+  if (isBrowserConnected()) return;
   if (initPromise) return initPromise;
 
   initPromise = (async () => {
     try {
-      if (!isBrowserConnected() || !context) {
+      if (!isBrowserConnected()) {
         logToDebugFile('INFO', `[Worker-${workerId}] Initializing browser instance...`);
 
         // ---------------------------------------------------------------------
@@ -196,8 +198,12 @@ export async function initBrowser(): Promise<void> {
         //
         // 90s launch timeout: CI runners (2 vCPU) need up to 60s to start Firefox
         // when it has both CPUs. The old 45s limit was too tight and caused silent
-        // init failures under resource pressure.
-        const launchTimeoutMs = 90000;
+        // init failures under resource pressure. Imported from config.ts (single
+        // source of truth) rather than a local literal — the scheduler's task
+        // deadlines (browser-task-scheduler.ts) and the healthcheck budget
+        // (getHealthCheckBudgetMs) both have to budget for this exact worst case,
+        // via COLD_START_ALLOWANCE_MS.
+        const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
         const launchOnce = async (headless: boolean | 'virtual') => {
           const launchPromise = Camoufox({
             headless,
@@ -244,11 +250,28 @@ export async function initBrowser(): Promise<void> {
           // timeout races ahead.
           launchPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser launch rejection: ${err.message}`));
           let launchTimeoutId: NodeJS.Timeout | undefined;
+          let raceLost = false;
+          // If the launch resolves AFTER we gave up on it, nobody holds the result:
+          // `browser` is assigned only from the race's winner, so the catch below —
+          // whose comment promises to "close any partially-launched browser" — finds
+          // null and closes nothing. Camoufox's own launch bound is 180s against our
+          // 90s, so a launch completing in that window leaves a fully-started headless
+          // Firefox with no owner, surviving the orphan sweep (its parent worker is
+          // alive) until the pool shuts down. Adopt and close it, exactly as
+          // createPageSafe already does for an orphaned page.
+          launchPromise.then((b: any) => {
+            if (!raceLost) return;
+            logToDebugFile('WARN', `[Worker-${workerId}] Browser launch completed after its deadline; closing the orphaned instance.`);
+            try { b?.close?.(); } catch { /* already gone */ }
+          }).catch(() => { /* rejection already logged above */ });
           try {
             return await Promise.race([
               launchPromise,
               new Promise<never>((_, reject) => {
-                launchTimeoutId = setTimeout(() => reject(new Error(`Browser launch timed out after ${launchTimeoutMs}ms`)), launchTimeoutMs);
+                launchTimeoutId = setTimeout(() => {
+                  raceLost = true;
+                  reject(new Error(`Browser launch timed out after ${launchTimeoutMs}ms`));
+                }, launchTimeoutMs);
               })
             ]);
           } finally {
@@ -278,22 +301,6 @@ export async function initBrowser(): Promise<void> {
         }
 
         browser = launchedBrowser;
-        // newContext() can hang if the browser process becomes unresponsive immediately
-        // after launch (e.g. OOM, GPU crash). Guard with a hard timeout.
-        const contextPromise = browser.newContext();
-        contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
-        let contextTimeoutId: NodeJS.Timeout | undefined;
-        context = await Promise.race([
-          contextPromise,
-          new Promise<never>((_, reject) => {
-            contextTimeoutId = setTimeout(() => reject(new Error('Browser context creation timed out after 30000ms')), 30000);
-          })
-        ]);
-        if (contextTimeoutId) clearTimeout(contextTimeoutId);
-        
-        // Setup mocking for CI if enabled
-        await setupMocking(context);
-        
         logToDebugFile('INFO', `[Worker-${workerId}] Browser initialized.`);
       }
     } catch (e: unknown) {
@@ -304,7 +311,6 @@ export async function initBrowser(): Promise<void> {
         Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during failed init: ${err.message}`));
       }
       browser = null;
-      context = null;
       const msg = e instanceof Error ? e.message : String(e);
 
       if (msg.includes('Camoufox is not installed') || msg.includes('Version information not found')) {
@@ -331,16 +337,60 @@ export async function initBrowser(): Promise<void> {
 }
 
 /**
- * Get the current browser context
+ * Create a FRESH BrowserContext for one task, against the shared browser
+ * process. Call once per task, right after initBrowser() resolves; the
+ * caller owns the returned context and must close it itself (in a `finally`,
+ * covering the success, error, AND timeout-abort paths) once the task ends.
+ *
+ * Never reused across tasks — that was the bug this replaced. Two tasks
+ * dispatched concurrently to the same worker (WORKER_CONCURRENCY > 1, the
+ * default) used to share one BrowserContext, so cookies/storage a site set
+ * for one task's request were visible to a different, unrelated task's later
+ * request to the same site — a content-consistency gap with no counterpart
+ * anywhere else in this codebase, which scopes browser-adjacent state
+ * (circuit breakers, scrape caches) per session deliberately.
+ *
+ * The launch race/timeout/orphan-adoption pattern below is the exact one
+ * initBrowser() used when context creation lived there (moved here wholesale,
+ * not reinvented): newContext() can hang if the browser process is
+ * unresponsive (OOM, GPU crash), so it is guarded with a hard timeout, and a
+ * context that finishes creating AFTER that timeout is adopted and closed
+ * rather than left to leak.
  */
-export function getContext(): any {
-  return context;
+export async function acquireTaskContext(): Promise<any> {
+  const contextPromise = browser.newContext();
+  contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
+  let contextTimeoutId: NodeJS.Timeout | undefined;
+  let contextRaceLost = false;
+  contextPromise.then((c: any) => {
+    if (!contextRaceLost) return;
+    logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
+    try { c?.close?.(); } catch { /* already gone */ }
+  }).catch(() => { /* rejection already logged above */ });
+  let newContext: any;
+  try {
+    newContext = await Promise.race([
+      contextPromise,
+      new Promise<never>((_, reject) => {
+        contextTimeoutId = setTimeout(() => {
+          contextRaceLost = true;
+          reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
+        }, CONTEXT_CREATION_TIMEOUT_MS);
+      })
+    ]);
+  } finally {
+    if (contextTimeoutId) clearTimeout(contextTimeoutId);
+  }
+
+  // Route interception is per-context, so mocking must be (re-)applied to
+  // every fresh context, not just the first one ever created.
+  await setupMocking(newContext);
+
+  return newContext;
 }
 
 function doResetBrowser(): void {
-  if (context) Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during reset: ${err.message}`));
   if (browser) Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during reset: ${err.message}`));
-  context = null;
   browser = null;
 }
 
@@ -377,7 +427,9 @@ export async function cleanupBrowser(): Promise<void> {
   const timeoutMs = 2000;
   
   const cleanup = async () => {
-    if (context) await Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during cleanup: ${err.message}`));
+    // Closes every context still open under it (any task whose own close()
+    // hasn't run yet) as a side effect — no separate context bookkeeping
+    // needed here now that contexts are task-owned, not module-owned.
     if (browser) await Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during cleanup: ${err.message}`));
   };
 
@@ -389,7 +441,6 @@ export async function cleanupBrowser(): Promise<void> {
   } catch {
     // Ignore timeout — we just want to ensure cleanup doesn't hang indefinitely
   } finally {
-    context = null;
     browser = null;
   }
 }

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -259,15 +259,42 @@ async function extractSearchResults(page: any): Promise<any[]> {
 /**
  * Execute a search task
  */
+/**
+ * Bind a task-timeout signal to the page it is bounding.
+ *
+ * `page.setDefaultTimeout` bounds Playwright's action and navigation verbs
+ * individually; it does not bound their sum, and it does not bound a Response read
+ * at all. Closing the page is what actually ends the work: any in-flight call
+ * rejects with 'Target closed', which runTask's catch already maps back to the
+ * task timeout. Fire-and-forget, `{ once }` so the listener cannot outlive the task.
+ */
+function bindTaskAbortToPage(page: any, signal?: AbortSignal): void {
+  if (!signal) return;
+  const onAbort = () => { page.close().catch(() => {}); };
+  if (signal.aborted) onAbort();
+  else signal.addEventListener('abort', onAbort, { once: true });
+}
+
 export async function executeSearchTask(
   _context: any,
-  query: string
+  query: string,
+  signal?: AbortSignal
 ): Promise<{ results: any[]; jitter: number }> {
   const page = await createPageSafe(_context);
   const SEARCH_TIMEOUT = parseTimeoutMs(process.env['PI_RESEARCH_SEARCH_TIMEOUT_MS'], 45000);
   page.setDefaultTimeout(SEARCH_TIMEOUT);
   page.setDefaultNavigationTimeout(SEARCH_TIMEOUT);
 
+  // Same binding the scrape path has used all along, and for the same reason: the
+  // worker's task deadline was armed but its signal was only ever handed to
+  // executeScrapeTask, so on a search it fired into nothing. Playwright's own
+  // per-action timeouts still bound each verb, but the sum of them plus the jitter
+  // sleep can outlast the whole task budget — and poolifier keeps counting this node
+  // as executing until the worker replies, so the node stays pinned long after the
+  // caller was answered. Closing the page rejects whatever is in flight with
+  // 'Target closed', which runTask's catch already recognises as the timeout.
+  bindTaskAbortToPage(page, signal);
+
   try {
     logToDebugFile('DEBUG', `[Worker-${workerId}] Starting search for: ${query}`);
     
@@ -406,6 +433,13 @@ export async function executeScrapeTask(
   let poisonedError: Error | null = null;
   const pendingAddrChecks: Array<Promise<void>> = [];
 
+  // Set by the response-side early-abort check (see page.on('response') below) when the
+  // main-frame document's OWN headers already declare a PDF larger than MAX_PDF_SIZE.
+  // Declared outside the try for the same reason as poisonedError: the catch below needs
+  // to surface this instead of the secondary 'Target closed' error the triggering
+  // page.close() produces.
+  let oversizedPdfError: Error | null = null;
+
   try {
     logToDebugFile('DEBUG', `[Worker-${workerId}] Starting scrape for: ${url}`);
 
@@ -484,6 +518,42 @@ export async function executeScrapeTask(
       try {
         if (resp.request().resourceType() === 'document' && resp.frame() === page.mainFrame()) {
           finalMainFrameAddr = resp.serverAddr().catch(() => null);
+
+          // Early-abort a declared-oversized PDF as soon as its headers arrive, rather
+          // than waiting for page.goto() to finish downloading the whole thing first.
+          // 'response' fires on headers-received, well before the body is fully
+          // transferred — for a direct navigation to a PDF URL, Playwright's goto()
+          // does not resolve until the download completes, so without this the full
+          // (possibly multi-hundred-MB) body is pulled into the browser process before
+          // the PDF branch below ever gets to check Content-Length.
+          //
+          // This closes the gap only for a server that HONESTLY declares an oversized
+          // Content-Length up front. A chunked-transfer response (no Content-Length) or
+          // one that understates its length and then sends far more cannot be caught
+          // this way — Playwright's public API exposes no incremental
+          // bytes-received/streaming hook to abort mid-body on either engine. That
+          // residual case is bounded instead by the task's own SCRAPE_TIMEOUT_MS +
+          // BROWSER_TASK_TIMEOUT_MS deadline (see the onAbort/page.close() wiring
+          // above), which forcibly closes the page — and with it any in-flight
+          // download — once the task's time budget is exhausted, converting an
+          // otherwise-unbounded read into one bounded by (bandwidth x that budget).
+          const check2: Promise<void> = Promise.resolve(resp.headerValue('content-type'))
+            .then(async (ct: string | null) => {
+              // Matches the PDF branch below AND the fetch layer's own detection
+              // (web-scraper.ts): a server can mislabel or omit content-type, so a
+              // `.pdf` URL extension is an equally-trusted signal, not just the header.
+              const looksLikePdf = ct?.includes('application/pdf') || url.toLowerCase().endsWith('.pdf');
+              if (!looksLikePdf || oversizedPdfError || poisonedError) return;
+              const declared = parseInt((await resp.headerValue('content-length').catch(() => null)) || '', 10);
+              if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+                const sizeMB = Math.round(declared / 1024 / 1024);
+                oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+                logToDebugFile('DEBUG', `[Worker-${workerId}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
+                page.close().catch(() => {});
+              }
+            })
+            .catch(() => {});
+          pendingAddrChecks.push(check2);
         }
         const check: Promise<void> = Promise.resolve(resp.serverAddr())
           .then((addr: { ipAddress?: string } | null) => {
@@ -535,7 +605,14 @@ export async function executeScrapeTask(
     // 403/503 — the HTML branch defers to the challenge wait below for that.
     const status = response?.status?.() ?? 0;
 
-    if (contentType.includes('application/pdf')) {
+    // `.pdf` extension is an equally-trusted signal alongside the header (matches
+    // the fetch layer's own detection in web-scraper.ts): a server that mislabels
+    // or omits content-type would otherwise fall into the HTML branch below, whose
+    // page.content() would serialize Firefox's PDF-viewer chrome instead of
+    // extracting the document — a silent content loss for exactly the URLs this
+    // feature advertises as supported, and specifically for the malformed/oddly-
+    // served PDFs most likely to reach this browser fallback in the first place.
+    if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
       // An errored "PDF" is an error page: fail BEFORE buffering its body.
       if (status >= 400) throw new Error(`HTTP ${status}`);
@@ -549,6 +626,10 @@ export async function executeScrapeTask(
         const sizeMB = Math.round(declaredLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
+      // The response-side early-abort check above (page.on('response')) may already have
+      // decided this and be in the middle of closing the page — skip the (redundant,
+      // about-to-fail) download attempt rather than race it.
+      if (oversizedPdfError) throw oversizedPdfError;
       const buffer = await response.body();
       // ...then the actual bytes: the header is advisory and absent on chunked
       // responses, so the byte count is the check that actually bounds memory.
@@ -665,9 +746,12 @@ export async function executeScrapeTask(
     return { contentType, html, jitter };
   } catch (error) {
     await page.close().catch((err: any) => logToDebugFile('DEBUG', `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
-    // A poison-triggered page.close() makes the in-flight goto/content() reject
-    // with 'Target closed'; report the SSRF poisoning (a benign-classified
-    // 'Fetch blocked:' failure), not the secondary teardown error.
+    // An early-abort-triggered page.close() (oversized PDF, or SSRF poisoning) makes
+    // the in-flight goto/content()/body() reject with 'Target closed'; report the
+    // actual reason (a benign-classified failure) instead of that secondary teardown
+    // error. oversizedPdfError takes priority since it can only be set on the PDF path,
+    // where poisonedError (an HTML-only DOM-exfil concern) is not meaningful.
+    if (oversizedPdfError) throw oversizedPdfError;
     if (poisonedError) throw poisonedError;
     throw error;
   } finally {
@@ -692,11 +776,15 @@ const HEALTH_FALLBACK_URL = 'https://example.com/';
 async function executeHealthCheckAttempt(
   _context: any,
   navTimeoutMs: number,
-  url: string = HEALTH_PRIMARY_URL
+  url: string = HEALTH_PRIMARY_URL,
+  signal?: AbortSignal
 ): Promise<{ success: boolean; navMs: number }> {
   const page = await createPageSafe(_context);
   page.setDefaultTimeout(navTimeoutMs);
   page.setDefaultNavigationTimeout(navTimeoutMs);
+  // See executeSearchTask: the probe runs twice (primary then fallback), so its own
+  // nav timeout bounds each attempt but not the pair.
+  bindTaskAbortToPage(page, signal);
 
   try {
     const navStart = Date.now();
@@ -721,7 +809,8 @@ async function executeHealthCheckAttempt(
  */
 export async function executeHealthCheck(
   _context: any,
-  initMs: number
+  initMs: number,
+  signal?: AbortSignal
 ): Promise<{ success: boolean; navMs: number }> {
   // Nav timeout: read from env (passed through getBrowserEnv), floor at 10s.
   // The outer BrowserTaskScheduler.runHealthCheck() has its own 45s hard deadline.
@@ -733,14 +822,14 @@ export async function executeHealthCheck(
   }
 
   try {
-    return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT);
+    return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_PRIMARY_URL, signal);
   } catch (firstError: unknown) {
     // One retry: if the first attempt fails, the page may have been in a bad state
     // (challenge redirect, partial load, transient network blip). A second attempt
     // on a fresh page helps distinguish a real outage from a one-off failure.
     logToDebugFile('WARN', `[Worker-${workerId}] Health check attempt 1 failed: ${firstError instanceof Error ? firstError.message : String(firstError)}. Retrying once...`);
     try {
-      return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT);
+      return await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_PRIMARY_URL, signal);
     } catch (retryError: unknown) {
       const retryMsg = retryError instanceof Error ? retryError.message : String(retryError);
       // A TIMEOUT is left to propagate: upstream (isBusyPoolHealthFailure) already
@@ -756,7 +845,7 @@ export async function executeHealthCheck(
         // healthy enough to research (search has its own retries/fallbacks downstream).
         logToDebugFile('WARN', `[Worker-${workerId}] Search-provider health probe failed twice (${retryMsg}); verifying open-web reachability via fallback endpoint...`);
         try {
-          const fallback = await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_FALLBACK_URL);
+          const fallback = await executeHealthCheckAttempt(_context, HEALTH_TIMEOUT, HEALTH_FALLBACK_URL, signal);
           logToDebugFile('WARN', `[Worker-${workerId}] Fallback endpoint reachable — open web is up, search provider is degraded. Proceeding with research.`);
           return fallback;
         } catch (fallbackError: unknown) {