@lincoln504/pi-research 1.2.0 → 1.3.0

package/src/infrastructure/browser/browser-task-scheduler.ts

@@ -202,9 +202,16 @@ export class BrowserTaskScheduler implements IScheduler {
 
     async startServer(): Promise<number> {
         this.server = new BrowserServer({
-            onSearch: (q) => this.runSearch(q),
-            onScrape: (u) => this.runScrape(u),
-            onHealthCheck: () => this.runHealthCheck(),
+            // The per-request signal aborts when the follower's socket closes
+            // before completion, so the leader-side task releases its queue
+            // claim (abort-while-queued/running is the PriorityTaskQueue's own
+            // pinned semantics) instead of holding the slot to full timeout.
+            // Healthcheck mirrors search/scrape here even though its one current
+            // production caller (src/healthcheck/index.ts) never passes a signal —
+            // the wiring must not silently drop one the moment a caller does.
+            onSearch: (q, signal) => this.runSearch(q, undefined, signal),
+            onScrape: (u, signal) => this.runScrape(u, undefined, signal),
+            onHealthCheck: (signal) => this.runHealthCheck(undefined, signal),
         });
         // FIX (#21): Expose auth secret to child processes via env
         process.env['PI_BROWSER_AUTH_SECRET'] = getBrowserServerAuthSecret();
@@ -273,6 +280,18 @@ export class BrowserTaskScheduler implements IScheduler {
                 logger.debug(`[BrowserTaskScheduler] Search abandoned during shutdown: "${query}"`);
                 throw error;
             }
+            // The caller's own signal aborting (follower socket closed / research run
+            // cancelled) is what PriorityTaskQueue's abort-while-queued/running
+            // rejection above reports. This is routine, expected cancellation, not a
+            // fault — cancelling a run routinely cancels its in-flight browser tasks,
+            // so this fires often. Checking the signal directly (rather than matching
+            // the queue's rejection text) mirrors how BrowserServer's own catch block
+            // and task-execution-service.ts already classify this. Demote to DEBUG so
+            // it doesn't inflate ERROR counts / browser_search_errors_total.
+            if (signal?.aborted) {
+                logger.debug(`[BrowserTaskScheduler] Search cancelled by caller: "${query}"`);
+                throw error;
+            }
             logger.error(`[BrowserTaskScheduler] Search failed: "${query}"`, error);
             metrics.increment('browser_search_errors_total', 1);
             errorTracker.trackError(error instanceof Error ? error : String(error), {
@@ -288,8 +307,10 @@ export class BrowserTaskScheduler implements IScheduler {
         }
 
         const duration = Date.now() - startTime;
-        metrics.observe('browser_search_duration_ms', duration, { status: 'success' });
-        metrics.increment('browser_search_requests_total', 1, { status: 'success' });
+        // In-band worker error checked FIRST: a task the worker reports as failed
+        // must count only as an error. Recording success unconditionally before
+        // this check double-counted every worker-reported failure as
+        // success+error, so dashboards read healthy through a full outage.
         if (result.error) {
             metrics.increment('browser_search_requests_total', 1, { status: 'error' });
             errorTracker.trackError(new Error(result.error), {
@@ -301,6 +322,8 @@ export class BrowserTaskScheduler implements IScheduler {
             });
             throw new Error(result.error);
         }
+        metrics.observe('browser_search_duration_ms', duration, { status: 'success' });
+        metrics.increment('browser_search_requests_total', 1, { status: 'success' });
         return result.results;
     }
 
@@ -347,6 +370,12 @@ export class BrowserTaskScheduler implements IScheduler {
                 logger.debug(`[BrowserTaskScheduler] Scrape abandoned during shutdown: "${url}"`);
                 throw error;
             }
+            // See runSearch: the caller's own signal aborting is routine cancellation
+            // (follower disconnect / run cancelled), not a fault — demote to DEBUG.
+            if (signal?.aborted) {
+                logger.debug(`[BrowserTaskScheduler] Scrape cancelled by caller: "${url}"`);
+                throw error;
+            }
             metrics.increment('browser_scrape_errors_total', 1);
             errorTracker.trackError(error instanceof Error ? error : String(error), {
                 component: 'browser-manager',
@@ -360,8 +389,7 @@ export class BrowserTaskScheduler implements IScheduler {
         }
 
         const duration = Date.now() - startTime;
-        metrics.observe('browser_scrape_duration_ms', duration, { status: 'success' });
-        metrics.increment('browser_scrape_requests_total', 1, { status: 'success' });
+        // In-band error first — success-side recording must not precede it (see runSearch).
         if (result.error) {
             metrics.increment('browser_scrape_requests_total', 1, { status: 'error' });
             errorTracker.trackError(new Error(result.error), {
@@ -372,6 +400,8 @@ export class BrowserTaskScheduler implements IScheduler {
             });
             throw new Error(result.error);
         }
+        metrics.observe('browser_scrape_duration_ms', duration, { status: 'success' });
+        metrics.increment('browser_scrape_requests_total', 1, { status: 'success' });
         return result;
     }
 
@@ -412,6 +442,13 @@ export class BrowserTaskScheduler implements IScheduler {
                 logger.debug(`[BrowserTaskScheduler] Healthcheck abandoned during shutdown`);
                 throw error;
             }
+            // See runSearch: the caller's own signal aborting is routine cancellation,
+            // not a fault — demote to DEBUG. Reachable now that startServer() forwards
+            // a signal into onHealthCheck (see BrowserServer's onHealthCheck wiring).
+            if (signal?.aborted) {
+                logger.debug(`[BrowserTaskScheduler] Healthcheck cancelled by caller`);
+                throw error;
+            }
             metrics.increment('browser_healthcheck_errors_total', 1);
             errorTracker.trackError(error instanceof Error ? error : String(error), {
                 component: 'browser-manager',
@@ -425,10 +462,10 @@ export class BrowserTaskScheduler implements IScheduler {
         }
 
         const duration = Date.now() - startTime;
-        metrics.observe('browser_healthcheck_duration_ms', duration, { status: 'success' });
-        metrics.increment('browser_healthcheck_requests_total', 1, { status: 'success' });
-        metrics.setGauge('browser_pool_health', 1);
-        logger.debug(`[Scheduler] Healthcheck completed in ${duration}ms`);
+        // In-band error first — success-side recording must not precede it (see
+        // runSearch). This path additionally owns the browser_pool_health gauge:
+        // setting it to 1 before the check flickered the pool healthy on every
+        // failed probe during an outage.
         if (result.error) {
             metrics.increment('browser_healthcheck_requests_total', 1, { status: 'error' });
             metrics.setGauge('browser_pool_health', 0);
@@ -440,6 +477,10 @@ export class BrowserTaskScheduler implements IScheduler {
             });
             throw new Error(result.error);
         }
+        metrics.observe('browser_healthcheck_duration_ms', duration, { status: 'success' });
+        metrics.increment('browser_healthcheck_requests_total', 1, { status: 'success' });
+        metrics.setGauge('browser_pool_health', 1);
+        logger.debug(`[Scheduler] Healthcheck completed in ${duration}ms`);
         return result;
     }
 
@@ -505,7 +546,12 @@ export class BrowserTaskScheduler implements IScheduler {
             logger.warn('[Scheduler] Could not read browser server state during shutdown:', err);
         }
         if (serverInfo?.pid === process.pid && serverInfo?.schedulerId === this.schedulerId) {
-            await this.stateManager.clearBrowserServer().catch((err) => {
+            // CAS: this belief-check (pid + schedulerId match) happens outside any
+            // lock, so pass it as `expected` rather than relying on it as an
+            // assumption — if a successor claimed the slot between the read above
+            // and this clear, the mismatch makes the clear a no-op instead of
+            // deregistering the new owner.
+            await this.stateManager.clearBrowserServer({ pid: process.pid, schedulerId: this.schedulerId }).catch((err) => {
                 logger.warn('[Scheduler] Failed to clear browser server state during shutdown:', err);
             });
         }

package/src/infrastructure/browser/cleanup-utils.ts

@@ -108,6 +108,48 @@ async function isProfileLive(profileDir: string): Promise<boolean> {
   return false;
 }
 
+/** Canonicalize a path for identity comparison: realpath when resolvable (macOS
+ *  /tmp is a symlink to /private/tmp), else a plain resolve; case-folded on
+ *  Windows, whose filesystem compares case-insensitively. */
+async function canonicalizeForCompare(p: string): Promise<string> {
+  let resolved: string;
+  try {
+    resolved = await fs.realpath(p);
+  } catch {
+    resolved = path.resolve(p);
+  }
+  return process.platform === 'win32' ? resolved.toLowerCase() : resolved;
+}
+
+/**
+ * May the unrecognized-entry deletion branch run against this explicit baseDir?
+ *
+ * That branch recursively rm's ANY stale subdirectory, so it is only safe inside
+ * a directory pi-research owns outright. Two conditions gate it: the dir must not
+ * BE the shared system temp dir (realpath-compared on both sides — a
+ * PI_RESEARCH_TMP_DIR pointed at /tmp once made every same-user /tmp entry — ssh
+ * agents, build caches — a sweep candidate), and the path must carry a
+ * pi-research segment as positive evidence of ownership. When this returns false
+ * the prefix-matched playwright profile cleanup still runs; only unknowns are
+ * spared.
+ */
+async function isPiOwnedSweepDir(dir: string): Promise<boolean> {
+  const [realDir, realTmp] = await Promise.all([
+    canonicalizeForCompare(dir),
+    canonicalizeForCompare(os.tmpdir()),
+  ]);
+  if (realDir === realTmp) return false;
+  const hasPiSegment = (p: string) =>
+    p.split(/[\\/]+/).some((segment) => segment.toLowerCase().startsWith('pi-research'));
+  // Only the CANONICAL path counts. A symlinked dir whose target lacks the
+  // marker is NOT sweep-safe: approving it by its configured name would send the
+  // recursive rm THROUGH the symlink into a directory we do not own (e.g. a
+  // /tmp/pi-research/profiles link pre-created by another user under a
+  // world-writable PI_RESEARCH_TMP_DIR). Unknowns are spared; the prefix-matched
+  // playwright profile cleanup still runs either way.
+  return hasPiSegment(realDir);
+}
+
 /**
  * Reclaim leaked Playwright/Camoufox profile directories.
  *
@@ -134,6 +176,11 @@ export async function cleanupStaleProfiles(
   let removed = 0;
   let errors = 0;
 
+  // Explicit-baseDir mode considers EVERY entry a candidate, including ones it
+  // does not recognize; deleting unknowns is only permitted inside a dir that is
+  // provably pi-owned and not the shared system temp dir (see isPiOwnedSweepDir).
+  const sweepUnrecognized = usePrefix ? false : await isPiOwnedSweepDir(scanDir);
+
   try {
     const entries = await fs.readdir(scanDir);
     const now = Date.now();
@@ -171,8 +218,9 @@ export async function cleanupStaleProfiles(
         }
 
         // Unrecognized entry (only reachable with an explicit baseDir): be conservative —
-        // only reclaim if both old AND not live, so we never delete something unknown promptly.
-        if (now - stats.mtimeMs > staleMs && !(await isProfileLive(fullPath))) {
+        // only reclaim inside a pi-owned dir, and only if both old AND not live,
+        // so nothing unknown is ever deleted promptly or outside our ownership.
+        if (sweepUnrecognized && now - stats.mtimeMs > staleMs && !(await isProfileLive(fullPath))) {
           await fs.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
           removed++;
         }

package/src/infrastructure/browser/client-agent.ts

@@ -10,17 +10,17 @@ import * as http from 'node:http';
 /**
  * Global HTTP Agent for high-concurrency client requests
  *
- * IMPORTANT: The socket timeout must be longer than the BrowserClient request timeout (60s,
- * REQUEST_TIMEOUT_MS in browser-client.ts) to prevent premature socket closure when the
- * browser is slow or the queue is deep.
+ * IMPORTANT: The socket timeout must exceed the largest possible BrowserClient
+ * request timeout (CLIENT_TIMEOUT_CAP_MS = 300s in browser-client.ts) so the
+ * request-level timer — whose error text is classified by the retry gates —
+ * always answers before the socket layer tears the connection down.
  */
 const clientAgent = new http.Agent({
     keepAlive: true,
     maxSockets: 100, // Allow up to 100 concurrent requests to the leader
     maxFreeSockets: 10,
-    // 180s (3x the client timeout) to handle slow browser responses
-    // and prevent "socket hang up" errors during peak load
-    timeout: 180000
+    // Above CLIENT_TIMEOUT_CAP_MS (300s) with margin — see the invariant above.
+    timeout: 330000
 });
 
 /**

package/src/infrastructure/browser/config.ts

@@ -61,8 +61,9 @@ export function getBrowserCacheDir(): string {
  * os.tmpdir(): on systems where /tmp is tmpfs (RAM-backed), placing several
  * browser profiles there consumes RAM and, with an OOM killer like earlyoom
  * active, can contribute to the whole session being killed. Override with
- * PI_RESEARCH_TMP_DIR (config.TMP_DIR) — e.g. point it back at the system temp
- * dir to deliberately use tmpfs/RAM when there is memory headroom.
+ * PI_RESEARCH_TMP_DIR (config.TMP_DIR) — e.g. point it at the system temp dir
+ * to deliberately use tmpfs/RAM when there is memory headroom; profiles then
+ * land in <TMP_DIR>/pi-research/profiles, never in TMP_DIR itself.
  *
  * The directory is created if missing, because Playwright requires the parent
  * of its mkdtemp profile dir to already exist.
@@ -71,7 +72,17 @@ export function getBrowserProfileDir(config?: Config): string {
     const configured = (config || getConfig()).TMP_DIR;
     let dir: string;
     if (configured && configured.trim().length > 0) {
-        dir = configured;
+        // Containment invariant: profiles live in a pi-research-owned
+        // SUBDIRECTORY of the configured dir, never the configured dir itself.
+        // The pool-startup sweep (cleanupStaleProfiles with an explicit baseDir)
+        // treats every entry of this directory as a reclaim candidate, so using
+        // PI_RESEARCH_TMP_DIR verbatim — e.g. pointed at /tmp per the tmpfs
+        // opt-in above — put unrelated same-user directories (ssh agents, build
+        // caches) on the sweep's candidate list. The pi-research/profiles shape
+        // also matches PI_BROWSER_MARKER's profile-path fallback in
+        // browser-cleanup.ts, so orphan detection keeps working for relocated
+        // profiles.
+        dir = join(configured, 'pi-research', 'profiles');
     } else {
         const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
         dir = join(cacheHome, 'pi-research', 'profiles');

package/src/infrastructure/browser/scheduler-factory.ts

@@ -132,7 +132,16 @@ export async function forceSchedulerRestart(forceClearRemoteState: boolean = fal
         }
 
         if (shouldClearState) {
-            await stateManager.clearBrowserServer().catch((error) => {
+            // Compare-and-delete against the registration we just read: without a
+            // lock spanning the read-decide-clear sequence above, a concurrent
+            // process can independently detect the same dead leader, complete its
+            // own (properly locked) election in getScheduler(), and publish a
+            // fresh registration in this exact window. An unconditional clear here
+            // would wipe that brand-new, legitimate leader — it keeps running but
+            // becomes undiscoverable via state, so the next caller elects a
+            // second, redundant leader. See StateBrowserManager.clearBrowserServer.
+            const expected = serverInfo ? { pid: serverInfo.pid, schedulerId: serverInfo.schedulerId } : undefined;
+            await stateManager.clearBrowserServer(expected).catch((error) => {
                 logger.warn('[Scheduler] Failed to clear browser server from state:', error);
             });
         }
@@ -222,12 +231,18 @@ export async function getScheduler(config?: Config, container: ServiceContainer
                         if (serverInfo.pid !== process.pid) {
                             logger.log(`[Scheduler] Bypassing stale scheduler process (PID ${serverInfo.pid}) by clearing state...`);
                         }
-                        await stateManager.clearBrowserServer();
+                        // Defense-in-depth CAS: this read-then-clear is entirely inside
+                        // the browser-init lock, which serializes other getScheduler()
+                        // callers, but callers that clear without taking this lock
+                        // (forceSchedulerRestart, scheduler shutdown) are not excluded —
+                        // pass what was just read so a mismatch no-ops instead of
+                        // deleting a registration this code didn't observe.
+                        await stateManager.clearBrowserServer({ pid: serverInfo.pid, schedulerId: serverInfo.schedulerId });
                     } else {
                         const portOk = await isPortListening(serverInfo.port);
                         if (!portOk) {
                             logger.warn(`[Scheduler] PID ${serverInfo.pid} is alive but port ${serverInfo.port} is not listening — clearing stale state and starting fresh.`);
-                            await stateManager.clearBrowserServer().catch((e) => {
+                            await stateManager.clearBrowserServer({ pid: serverInfo.pid, schedulerId: serverInfo.schedulerId }).catch((e) => {
                                 logger.warn('[Scheduler] Failed to clear stale browser server state:', e);
                             });
                         } else {

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -8,6 +8,7 @@ let workerId: string = '';
 
 import { appendFileSync } from 'node:fs';
 import { redactSecrets } from '../../utils/log-utils.ts';
+import { MAX_HTML_SIZE, MAX_PDF_SIZE } from '../../web-research/scraper-types.ts';
 
 /**
  * Mutex lock to serialize page creation. Playwright/Firefox can deadlock if newPage()
@@ -311,7 +312,7 @@ export async function executeScrapeTask(
   _context: any,
   url: string,
   signal?: AbortSignal
-): Promise<{ contentType: string; html?: string; buffer?: Buffer; jitter: number }> {
+): Promise<{ contentType: string; html?: string; bufferB64?: string; jitter: number }> {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseInt(process.env['PI_RESEARCH_SCRAPE_TIMEOUT_MS'] || '15000', 10);
   page.setDefaultTimeout(SCRAPE_TIMEOUT);
@@ -458,15 +459,47 @@ export async function executeScrapeTask(
     }
 
     const contentType = (await response?.headerValue('content-type')) || '';
+    // HTTP status of the final navigation response. Never consulting it let
+    // 404/500/403 pages return as normal {contentType, html} results — major
+    // sites' rich error pages cleared the stub gate, were cached as scrape
+    // SUCCESS, and became citable. The fetch layer enforces !response.ok; this
+    // layer must match it (`HTTP <status>` is benign-classified per URL). The
+    // one legitimate exception is a Cloudflare challenge, which arrives with
+    // 403/503 — the HTML branch defers to the challenge wait below for that.
+    const status = response?.status?.() ?? 0;
 
     if (contentType.includes('application/pdf')) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
+      // An errored "PDF" is an error page: fail BEFORE buffering its body.
+      if (status >= 400) throw new Error(`HTTP ${status}`);
+      // Enforce the parent's PDF cap HERE, not only after the IPC transfer:
+      // buffering a multi-hundred-MB PDF (plus the base64 copy below) in the
+      // worker can OOM-kill the process — stranding every sibling task — or hit
+      // ERR_STRING_TOO_LONG. Content-Length first (free when present, and it
+      // skips the read entirely)...
+      const declaredLength = parseInt((await response.headerValue('content-length').catch(() => null)) || '', 10);
+      if (Number.isFinite(declaredLength) && declaredLength > MAX_PDF_SIZE) {
+        const sizeMB = Math.round(declaredLength / 1024 / 1024);
+        throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+      }
       const buffer = await response.body();
+      // ...then the actual bytes: the header is advisory and absent on chunked
+      // responses, so the byte count is the check that actually bounds memory.
+      // Checked BEFORE toString('base64') doubles the allocation.
+      if (buffer.byteLength > MAX_PDF_SIZE) {
+        const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
+        throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+      }
       // Discard the bytes if any observed response came from a private address.
       await Promise.allSettled(pendingAddrChecks);
       if (poisonedError) throw poisonedError;
       await page.close();
-      return { contentType, buffer, jitter: 0 };
+      // Base64, never a raw Buffer: this result crosses the poolifier cluster IPC
+      // channel (default JSON serialization), where a Buffer arrives as
+      // {type:'Buffer',data:[...]} — and the follower path adds a second JSON hop
+      // through the leader's browser-server. A base64 string survives both hops
+      // verbatim (and is ~3x smaller on the wire than the data-array form).
+      return { contentType, bufferB64: buffer.toString('base64'), jitter: 0 };
     }
 
     // If it's HTML, check if we need to wait longer (JS-heavy sites)
@@ -476,6 +509,15 @@ export async function executeScrapeTask(
     const cfPatterns = ['_cf_chl_', 'cdn-cgi/challenge-platform', 'cf_chl_opt', 'Just a moment...', 'Checking your browser before accessing'];
     const hasCloudflare = cfPatterns.some((pattern: string) => html.includes(pattern));
 
+    // HTTP-error enforcement for HTML (see the PDF branch above). A Cloudflare
+    // challenge legitimately arrives with 403/503 and the wait below may clear
+    // it to the real page, so a challenged response defers to that wait — whose
+    // failure path already throws its own 'Fetch blocked:' error. Everything
+    // else with a 4xx/5xx status is an error page, not content.
+    if (status >= 400 && !hasCloudflare) {
+      throw new Error(`HTTP ${status}`);
+    }
+
     if (hasCloudflare) {
       logToDebugFile('WARN', `[Worker-${workerId}] Cloudflare challenge detected for: ${url}`);
 
@@ -521,6 +563,16 @@ export async function executeScrapeTask(
       html = await page.content();
     }
 
+    // Enforce the parent's rendered-HTML cap worker-side, BEFORE the string
+    // crosses JSON IPC (and, for followers, a second JSON hop through the
+    // leader's browser-server). The parent's identical check in web-scraper.ts
+    // runs only after the transfer — too late to protect this process from a
+    // pathological/hostile page ballooning worker memory. Same error wording.
+    if (html.length > MAX_HTML_SIZE) {
+      const sizeMB = Math.round(html.length / 1024 / 1024);
+      throw new Error(`Browser HTML too large (${sizeMB}MB, max 25MB)`);
+    }
+
     // Skip jitter in mock mode; in real browsing, add 500–1500ms to mimic human behavior
     const jitter = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true'
       ? 0

package/src/infrastructure/browser/thread-worker-types.ts

@@ -19,7 +19,9 @@ export interface TaskResult {
   error?: string;
   success?: boolean;
   navMs?: number;
-  buffer?: Buffer;
+  /** PDF bytes, base64-encoded: a raw Buffer would be mangled to
+   *  {type:'Buffer',data:[...]} by the cluster-IPC JSON serialization. */
+  bufferB64?: string;
   html?: string;
   contentType?: string;
 }