@lincoln504/pi-research 1.1.2 → 1.3.0

package/src/infrastructure/browser/task-execution-service.ts

@@ -17,6 +17,9 @@ import { getServiceContainer, getService } from '../../core/service-registry.ts'
 import type { ServiceContainer } from '../../core/service-registry.ts';
 import { ServiceNames } from '../../core/service-interfaces.ts';
 import type { ISchedulerFactory, IScheduler } from '../../core/scheduler-factory.ts';
+import type { ISchedulerInternals } from '../../core/interfaces/scheduler-interfaces.ts';
+import type { IStateManager } from '../../core/interfaces/state-manager-interfaces.ts';
+import { BrowserClient } from './browser-client.ts';
 
 // Cooldown to prevent cascading scheduler restarts (thundering herd)
 let lastRestartTime = 0;
@@ -62,6 +65,92 @@ async function recoverFromLeaderHandover(container: ServiceContainer): Promise<v
     );
 }
 
+/**
+ * Is the leader currently registered in state still a live process — and is it
+ * the one our cached handle actually targets?
+ *
+ * Used only to decide whether the restart cooldown below may be honoured.
+ * Anything that prevents a confident "yes" (no registered leader, an unreadable
+ * state file, a service-resolution failure, a cached client aimed at a different
+ * port than the registered leader's) answers `false`, because the cost of an
+ * unnecessary re-resolve is one cheap round-trip while the cost of skipping a
+ * needed one is a task failure against a dead port.
+ */
+async function isRegisteredLeaderAliveUncached(container: ServiceContainer): Promise<boolean> {
+    try {
+        const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, undefined, container);
+        const info = await stateManager.getBrowserServer();
+        if (!info) return false;
+        // "Leader alive" is not enough: the cooldown protects the CACHED handle, so
+        // that handle must also target the registered leader. If our leader B died
+        // and another process already elected C, C is alive — but our cached client
+        // still aims at B's dead port, and honouring the cooldown would leave every
+        // retry failing against the corpse until the window expired.
+        const schedulerService = await getService<ISchedulerInternals>(ServiceNames.SCHEDULER, undefined, container);
+        const cached = schedulerService.getSchedulerInstance();
+        if (cached instanceof BrowserClient && cached.targetPort !== info.port) return false;
+        return await stateManager.isPidAlive(info.pid, info.schedulerId);
+    } catch {
+        return false;
+    }
+}
+
+/**
+ * Coalesce concurrent liveness probes.
+ *
+ * This runs on the failure path, where a whole query burst fails at once: a
+ * 20-query search losing its leader calls it up to 19 times within milliseconds
+ * (the first caller restarts and the rest land inside the cooldown). The probe is
+ * cheap on Linux — a `/proc/<pid>/stat` read — but on Windows the PID-reuse guard
+ * resolves the process start time by spawning `powershell -NoProfile`, so an
+ * uncoalesced burst would fan out into ~19 concurrent PowerShell processes on the
+ * slowest platform we support. The window is deliberately far shorter than
+ * RESTART_COOLDOWN_MS: it exists to collapse one burst, not to cache a verdict.
+ */
+const LEADER_LIVENESS_COALESCE_MS = 1000;
+let leaderLivenessProbe: { startedAt: number; result: Promise<boolean> } | null = null;
+
+async function isRegisteredLeaderAlive(container: ServiceContainer): Promise<boolean> {
+    const now = Date.now();
+    if (leaderLivenessProbe && now - leaderLivenessProbe.startedAt < LEADER_LIVENESS_COALESCE_MS) {
+        return leaderLivenessProbe.result;
+    }
+    const result = isRegisteredLeaderAliveUncached(container);
+    leaderLivenessProbe = { startedAt: now, result };
+    return result;
+}
+
+/** Test seam: drop the coalesced liveness probe. */
+export function _resetLeaderLivenessProbeForTests(): void {
+    leaderLivenessProbe = null;
+}
+
+/**
+ * Drop the scheduler handle and wait for the pool to settle, subject to a
+ * cooldown that stops N concurrent task failures from each restarting.
+ *
+ * The cooldown is only safe while the registered leader is still alive: then the
+ * cached handle points at a working process and skipping the restart costs
+ * nothing. It is NOT safe when the leader is gone. A hard leader kill (SIGKILL,
+ * OOM) surfaces to clients as ECONNREFUSED, which `isTransientSocketError`
+ * matches but `isPoolShutdownError` does not — so it lands here rather than on
+ * the handover path, and a bare wait leaves the cached client aimed at a corpse.
+ * The retry then re-resolves to the same dead port and fails identically: the
+ * exact stale-handle failure the leader-handover work fixed for the *graceful*
+ * shutdown case. So the cooldown yields whenever the leader is not confirmed alive.
+ */
+async function restartSchedulerWithHerdGuard(container: ServiceContainer): Promise<void> {
+    const now = Date.now();
+    if (now - lastRestartTime > RESTART_COOLDOWN_MS || !(await isRegisteredLeaderAlive(container))) {
+        lastRestartTime = now;
+        logger.warn('[BrowserManager] Forcing scheduler restart and retrying...');
+        await forceSchedulerRestart(false, container);
+    } else {
+        logger.warn('[BrowserManager] Scheduler restart triggered recently and the registered leader is still alive — waiting for pool idle instead...');
+    }
+    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
+}
+
 /**
  * Get the scheduler from the factory, optionally using a specific container.
  */
@@ -163,16 +252,7 @@ export async function runBrowserTask<T>(
                 // PID+start-time liveness probe confirms the registered leader is
                 // actually gone; it only skips clearing when that probe finds the
                 // leader alive, which is exactly the case we must not disturb.
-                const now = Date.now();
-                if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
-                    lastRestartTime = now;
-                    logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
-                    await forceSchedulerRestart(false, container);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                } else {
-                    logger.warn(`[BrowserManager] Scheduler restart recently triggered, waiting for pool idle...`);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                }
+                await restartSchedulerWithHerdGuard(container);
             }
 
             // Retry with jitter (100-500ms) to prevent thundering herd
@@ -211,15 +291,7 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
                 // See runBrowserTask: don't force-clear remote leader state on a bare
                 // transient socket error — let the liveness probe inside
                 // forceSchedulerRestart decide.
-                const now = Date.now();
-                if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
-                    lastRestartTime = now;
-                    logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
-                    await forceSchedulerRestart(false, container);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                } else {
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                }
+                await restartSchedulerWithHerdGuard(container);
             }
 
             const jitter = 100 + Math.floor(Math.random() * 400);
@@ -278,15 +350,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
                 // See runBrowserTask: don't force-clear remote leader state on a bare
                 // transient socket error — let the liveness probe inside
                 // forceSchedulerRestart decide.
-                const now = Date.now();
-                if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
-                    lastRestartTime = now;
-                    logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
-                    await forceSchedulerRestart(false, container);
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                } else {
-                    await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
-                }
+                await restartSchedulerWithHerdGuard(container);
             }
 
             const jitter = 100 + Math.floor(Math.random() * 400);

package/src/infrastructure/browser/thread-worker-lifecycle.ts

@@ -206,14 +206,60 @@ export function setupOrphanProtection(): void {
     return;
   }
 
-  if (!process.ppid) {
+  // Capture the parent PID ONCE, at setup. `process.ppid` is re-read from the OS
+  // on every access, so on POSIX it changes the moment we are orphaned — and that
+  // change, not a liveness probe, is the reliable signal (see checkOrphan).
+  const originalPpid = process.ppid;
+  if (!originalPpid) {
     return;
   }
 
+  /** Tear down the browser and exit. Shared by both orphan signals below. */
+  const exitAsOrphan = async () => {
+    // Await cleanup to prevent browser/context leaks — but bounded. The browser is
+    // the plausible thing to be wedged after the parent died abruptly, and an
+    // unbounded browser.close() hang here would keep the orphan (and its browser)
+    // alive forever: the precise leak this guard exists to stop. The exit deadline
+    // wins over a perfect teardown.
+    if (cleanupBrowser) {
+      await Promise.race([
+        cleanupBrowser().catch(err => {
+          logToDebugFile('WARN', `[Worker-${workerId}] Browser cleanup failed during orphan exit:`, err);
+        }),
+        new Promise<void>(resolve => {
+          const t = setTimeout(resolve, 10_000);
+          if (t.unref) t.unref();
+        }),
+      ]);
+    }
+    // Clear the orphan check timer
+    cleanupOrphanProtection();
+    process.env['PI_PROCESS_EXITING'] = '1';
+    process.exit(1);
+  };
+
   const checkOrphan = async () => {
     try {
-      // signal 0 checks if the process is alive
-      process.kill(process.ppid, 0);
+      // Re-parenting is the authoritative orphan signal on POSIX. When the parent
+      // dies the kernel immediately re-parents us — to PID 1, or to the nearest
+      // subreaper (a user systemd instance, a container init). Probing the CURRENT
+      // ppid cannot see that: PID 1 always exists, and signalling it from an
+      // unprivileged process raises EPERM, which the handler below correctly reads
+      // as "alive but not ours to signal". Net effect on Linux was that this guard
+      // never fired at all and orphaned workers kept their browsers running.
+      // Comparing against the PID captured at setup detects it directly.
+      // Windows does not re-parent, so the liveness probe below remains the
+      // mechanism there and this check simply never trips.
+      if (process.ppid !== originalPpid) {
+        logToDebugFile('WARN', `[Worker-${workerId}] Re-parented (${originalPpid} -> ${process.ppid}): parent died, shutting down...`);
+        await exitAsOrphan();
+        return;
+      }
+
+      // signal 0 checks if the process is alive. Probe the ORIGINAL ppid, never
+      // `process.ppid` — after a re-parent the latter points at init, not at the
+      // process whose liveness we actually care about.
+      process.kill(originalPpid, 0);
 
       // Schedule next check
       orphanCheckTimer = setTimeout(checkOrphan, 10000);
@@ -231,16 +277,7 @@ export function setupOrphanProtection(): void {
       }
       // If error is thrown, the parent process is likely dead or unreachable
       logToDebugFile('WARN', `[Worker-${workerId}] Parent process died or unreachable (orphaned), shutting down...`);
-      // FIX: Await cleanup to prevent browser/context leaks
-      if (cleanupBrowser) {
-        await cleanupBrowser().catch(err => {
-          logToDebugFile('WARN', `[Worker-${workerId}] Browser cleanup failed during orphan exit:`, err);
-        });
-      }
-      // Clear the orphan check timer
-      cleanupOrphanProtection();
-      process.env['PI_PROCESS_EXITING'] = '1';
-      process.exit(1);
+      await exitAsOrphan();
     }
   };
 

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -8,6 +8,7 @@ let workerId: string = '';
 
 import { appendFileSync } from 'node:fs';
 import { redactSecrets } from '../../utils/log-utils.ts';
+import { MAX_HTML_SIZE, MAX_PDF_SIZE } from '../../web-research/scraper-types.ts';
 
 /**
  * Mutex lock to serialize page creation. Playwright/Firefox can deadlock if newPage()
@@ -311,7 +312,7 @@ export async function executeScrapeTask(
   _context: any,
   url: string,
   signal?: AbortSignal
-): Promise<{ contentType: string; html?: string; buffer?: Buffer; jitter: number }> {
+): Promise<{ contentType: string; html?: string; bufferB64?: string; jitter: number }> {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseInt(process.env['PI_RESEARCH_SCRAPE_TIMEOUT_MS'] || '15000', 10);
   page.setDefaultTimeout(SCRAPE_TIMEOUT);
@@ -458,15 +459,47 @@ export async function executeScrapeTask(
     }
 
     const contentType = (await response?.headerValue('content-type')) || '';
+    // HTTP status of the final navigation response. Never consulting it let
+    // 404/500/403 pages return as normal {contentType, html} results — major
+    // sites' rich error pages cleared the stub gate, were cached as scrape
+    // SUCCESS, and became citable. The fetch layer enforces !response.ok; this
+    // layer must match it (`HTTP <status>` is benign-classified per URL). The
+    // one legitimate exception is a Cloudflare challenge, which arrives with
+    // 403/503 — the HTML branch defers to the challenge wait below for that.
+    const status = response?.status?.() ?? 0;
 
     if (contentType.includes('application/pdf')) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
+      // An errored "PDF" is an error page: fail BEFORE buffering its body.
+      if (status >= 400) throw new Error(`HTTP ${status}`);
+      // Enforce the parent's PDF cap HERE, not only after the IPC transfer:
+      // buffering a multi-hundred-MB PDF (plus the base64 copy below) in the
+      // worker can OOM-kill the process — stranding every sibling task — or hit
+      // ERR_STRING_TOO_LONG. Content-Length first (free when present, and it
+      // skips the read entirely)...
+      const declaredLength = parseInt((await response.headerValue('content-length').catch(() => null)) || '', 10);
+      if (Number.isFinite(declaredLength) && declaredLength > MAX_PDF_SIZE) {
+        const sizeMB = Math.round(declaredLength / 1024 / 1024);
+        throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+      }
       const buffer = await response.body();
+      // ...then the actual bytes: the header is advisory and absent on chunked
+      // responses, so the byte count is the check that actually bounds memory.
+      // Checked BEFORE toString('base64') doubles the allocation.
+      if (buffer.byteLength > MAX_PDF_SIZE) {
+        const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
+        throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+      }
       // Discard the bytes if any observed response came from a private address.
       await Promise.allSettled(pendingAddrChecks);
       if (poisonedError) throw poisonedError;
       await page.close();
-      return { contentType, buffer, jitter: 0 };
+      // Base64, never a raw Buffer: this result crosses the poolifier cluster IPC
+      // channel (default JSON serialization), where a Buffer arrives as
+      // {type:'Buffer',data:[...]} — and the follower path adds a second JSON hop
+      // through the leader's browser-server. A base64 string survives both hops
+      // verbatim (and is ~3x smaller on the wire than the data-array form).
+      return { contentType, bufferB64: buffer.toString('base64'), jitter: 0 };
     }
 
     // If it's HTML, check if we need to wait longer (JS-heavy sites)
@@ -476,6 +509,15 @@ export async function executeScrapeTask(
     const cfPatterns = ['_cf_chl_', 'cdn-cgi/challenge-platform', 'cf_chl_opt', 'Just a moment...', 'Checking your browser before accessing'];
     const hasCloudflare = cfPatterns.some((pattern: string) => html.includes(pattern));
 
+    // HTTP-error enforcement for HTML (see the PDF branch above). A Cloudflare
+    // challenge legitimately arrives with 403/503 and the wait below may clear
+    // it to the real page, so a challenged response defers to that wait — whose
+    // failure path already throws its own 'Fetch blocked:' error. Everything
+    // else with a 4xx/5xx status is an error page, not content.
+    if (status >= 400 && !hasCloudflare) {
+      throw new Error(`HTTP ${status}`);
+    }
+
     if (hasCloudflare) {
       logToDebugFile('WARN', `[Worker-${workerId}] Cloudflare challenge detected for: ${url}`);
 
@@ -521,6 +563,16 @@ export async function executeScrapeTask(
       html = await page.content();
     }
 
+    // Enforce the parent's rendered-HTML cap worker-side, BEFORE the string
+    // crosses JSON IPC (and, for followers, a second JSON hop through the
+    // leader's browser-server). The parent's identical check in web-scraper.ts
+    // runs only after the transfer — too late to protect this process from a
+    // pathological/hostile page ballooning worker memory. Same error wording.
+    if (html.length > MAX_HTML_SIZE) {
+      const sizeMB = Math.round(html.length / 1024 / 1024);
+      throw new Error(`Browser HTML too large (${sizeMB}MB, max 25MB)`);
+    }
+
     // Skip jitter in mock mode; in real browsing, add 500–1500ms to mimic human behavior
     const jitter = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true'
       ? 0

package/src/infrastructure/browser/thread-worker-types.ts

@@ -19,7 +19,9 @@ export interface TaskResult {
   error?: string;
   success?: boolean;
   navMs?: number;
-  buffer?: Buffer;
+  /** PDF bytes, base64-encoded: a raw Buffer would be mangled to
+   *  {type:'Buffer',data:[...]} by the cluster-IPC JSON serialization. */
+  bufferB64?: string;
   html?: string;
   contentType?: string;
 }