@lincoln504/pi-research 1.7.8 → 1.7.10

package/src/infrastructure/browser/ensure-browser.ts

@@ -13,28 +13,28 @@
  *
  * Non-regression guarantee for the other install paths: when the browser is
  * already present — which it always is after the pi-extension / plain-npm
- * postinstall fetch — this is a cheap `existsSync` check that returns
- * immediately and changes nothing. It only ever does work when the binary is
- * genuinely absent.
+ * postinstall fetch — this is a cheap layout check that returns
+ * immediately and changes nothing. It only ever does work when the paired
+ * browser build is genuinely absent (or a legacy flat install needs migrating).
  */
 
 import { spawn } from 'node:child_process';
 import { createRequire } from 'node:module';
 import { createHash } from 'node:crypto';
-import { existsSync, readdirSync, statSync, fstatSync, openSync, closeSync, rmSync } from 'node:fs';
+import { existsSync, fstatSync, openSync, closeSync, rmSync, readFileSync, createWriteStream } from 'node:fs';
 import { join, dirname } from 'node:path';
-import { tmpdir } from 'node:os';
+import { tmpdir, platform, homedir } from 'node:os';
 import { logger } from '../../logger.ts';
 import { getCamoufoxBinaryPath } from './config.ts';
+import { resolveInstall, isUsable, provisionBrowser, loadPin, findPackageRoot, type BrowserPin } from './camoufox-layout.ts';
 
-/** Bound the ~500MB camoufox-js 0.12 browser download (measured 484MB; the
- *  0.10 pin was ~100MB and the old bound assumed that) so a stalled network
- *  fails eventually instead of hanging forever, while a slow-but-working link
- *  can actually finish. */
+/** Bound the ~1.3GB Camoufox browser download (the paired build is ~1.29 GB) so a stalled
+ *  network fails eventually instead of hanging forever, while a slow-but-working link can
+ *  actually finish. */
 const FETCH_TIMEOUT_MS = 45 * 60 * 1000;
 /** A lock older than this is considered stale (crashed mid-fetch) and may be stolen.
  *  Must exceed FETCH_TIMEOUT_MS: stealing a LIVE fetcher's lock mid-download would
- *  start a second 500MB download beside the first. */
+ *  start a second download beside the first. */
 const STALE_LOCK_MS = 55 * 60 * 1000;
 /** When another process holds the fetch lock, poll this long for the browser to appear.
  *  Spans a full FETCH_TIMEOUT_MS fetch, so a legitimate slow peer is waited out rather
@@ -45,68 +45,104 @@ const POLL_INTERVAL_MS = 2000;
 /** In-process dedupe: concurrent pool inits / scrapers share one fetch. */
 let inFlight: Promise<void> | null = null;
 
+/** The paired browser build this installed @camoufox/camoufox was released with. */
+function currentPin(): BrowserPin | null {
+  const root = findPackageRoot(import.meta.url);
+  return root ? loadPin(root) : null;
+}
+
 /**
- * True when a camoufox browser binary is actually present (cache dir exists and
- * contains at least one version subdirectory) — mirrors the check in setup.cjs.
+ * True when a usable Camoufox browser is installed: the new layout's compat flag plus
+ * the build this launcher is paired with (any build when the user chose one explicitly).
+ * A bare "directory exists" is NOT enough: a legacy flat camoufox-js install is a
+ * non-empty directory too, and the launcher deletes it on first use.
  */
 export function isBrowserBinaryPresent(): boolean {
   try {
-    const cacheDir = getCamoufoxBinaryPath();
-    if (!existsSync(cacheDir)) return false;
-    return readdirSync(cacheDir).some((entry) => {
-      try {
-        return statSync(join(cacheDir, entry)).isDirectory();
-      } catch {
-        return false;
-      }
-    });
+    return isUsable(getCamoufoxBinaryPath(), currentPin());
   } catch {
     return false;
   }
 }
 
-/** Resolve the camoufox-js CLI bin, mirroring scripts/setup.cjs (handles hoisting). */
-function resolveCamoufoxBin(): string | null {
+/** Resolve the `camoufox` CLI entry (JS file) of @camoufox/camoufox, mirroring scripts/setup.cjs. */
+function resolveCamoufoxCli(): string | null {
   try {
     const require = createRequire(import.meta.url);
-    const pkgJson = require.resolve('camoufox-js/package.json');
-    const bin = join(dirname(pkgJson), '..', '.bin', process.platform === 'win32' ? 'camoufox-js.cmd' : 'camoufox-js');
-    if (existsSync(bin)) return bin;
+    const pkgJson = require.resolve('@camoufox/camoufox/package.json');
+    const pkg = JSON.parse(readFileSync(pkgJson, 'utf8')) as { bin?: string | Record<string, string> };
+    const rel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.['camoufox'];
+    if (rel) {
+      const entry = join(dirname(pkgJson), rel);
+      if (existsSync(entry)) return entry;
+    }
   } catch {
-    /* fall through to npx */
+    /* fall through */
   }
   return null;
 }
 
 const delay = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
 
-/** Run `camoufox-js fetch`, resolving on exit 0 and rejecting otherwise / on timeout. */
-function runFetch(): Promise<void> {
-  return new Promise<void>((resolve, reject) => {
-    const bin = resolveCamoufoxBin();
-    const [cmd, args] = bin ? [bin, ['fetch']] : ['npx', ['camoufox-js', 'fetch']];
-    logger.info(`[ensure-browser] Camoufox not found; fetching the browser (this runs once, ~500MB): ${cmd} ${args.join(' ')}`);
-
-    // Windows needs shell:true to run the .cmd shim (Node CVE-2024-27980 fix makes a
-    // shell-less spawn of .cmd throw EINVAL), but shell:true performs NO quoting, so a
-    // spaced install path (C:\Users\John Smith\…) must be quoted explicitly. The args
-    // here are fixed literals ('fetch'), never user input, so quoting the command alone
-    // is sufficient and this stays injection-free.
-    const useShell = process.platform === 'win32';
-    const child = spawn(useShell ? `"${cmd}"` : cmd, args, { stdio: 'inherit', shell: useShell });
+/**
+ * Run `camoufox fetch`, resolving true on exit 0, false on any other exit, rejecting on
+ * spawn error / timeout. Runs the CLI's JS entry with process.execPath and an argv array:
+ * no shell, so neither a spaced Windows path nor a metacharacter in an install directory
+ * can be reinterpreted.
+ *
+ * The child's stdout is captured in a temp file, not inherited: a child with an
+ * inherited fd 1 writes raw to the terminal, which inside a pi session bypasses the
+ * host's JS-level stdout redirection (pi redirects `process.stdout.write` for its TUI)
+ * and interleaves the download's progress with the host's output stream — in
+ * print/JSON/RPC modes stdout is the protocol itself, so raw child output corrupts it.
+ * The launcher's progress is noise either way; its errors go to stderr, which stays
+ * inherited and is therefore still visible.
+ */
+function runFetch(): Promise<boolean> {
+  return new Promise<boolean>((resolve, reject) => {
+    const cli = resolveCamoufoxCli();
+    if (!cli) {
+      reject(new Error('@camoufox/camoufox is not installed (cannot find its CLI); run npm install'));
+      return;
+    }
+    // The launcher reads only XDG_CACHE_HOME / LOCALAPPDATA: hand it the override derived
+    // from the user's CAMOUFOX_INSTALL_DIR / PLAYWRIGHT_BROWSERS_PATH so the fetch lands
+    // where the lookup (getCamoufoxBinaryPath) expects it.
+    const install = resolveInstall(process.env, platform(), homedir());
+    const env: NodeJS.ProcessEnv = { ...process.env, ...install.overrides };
+    delete env['CAMOUFOX_INSTALL_DIR'];
+    delete env['PLAYWRIGHT_BROWSERS_PATH'];
+    logger.info(`[ensure-browser] Camoufox not found; fetching the browser (this runs once, ~1.3GB): ${process.execPath} ${cli} fetch`);
+
+    const outLogPath = join(tmpdir(), `pi-research-camoufox-fetch-${process.pid}-${Date.now()}.log`);
+    const outStream = createWriteStream(outLogPath, { mode: 0o600 });
+    const child = spawn(process.execPath, [cli, 'fetch'], { stdio: ['ignore', 'pipe', 'inherit'], env });
+    if (child.stdout) {
+      child.stdout.pipe(outStream);
+    }
     const timer = setTimeout(() => {
       child.kill('SIGKILL');
       reject(new Error(`camoufox fetch timed out after ${FETCH_TIMEOUT_MS}ms`));
     }, FETCH_TIMEOUT_MS);
 
-    child.on('error', (err) => {
+    const finish = (err: Error | null, ok: boolean): void => {
       clearTimeout(timer);
-      reject(err);
+      void outStream.close();
+      if (err) reject(err);
+      else resolve(ok);
+    };
+
+    child.on('error', (err) => {
+      const message = err instanceof Error ? err.message : String(err);
+      finish(new Error(`${message} (fetch output logged at ${outLogPath})`), false);
     });
     child.on('exit', (code) => {
-      clearTimeout(timer);
-      if (code === 0) resolve();
-      else reject(new Error(`camoufox fetch exited with code ${code}`));
+      if (code === 0) {
+        try { rmSync(outLogPath, { force: true }); } catch { /* best-effort */ }
+        finish(null, true);
+      } else {
+        finish(new Error(`camoufox fetch exited with code ${code} (fetch output logged at ${outLogPath})`), false);
+      }
     });
   });
 }
@@ -175,12 +211,14 @@ async function provision(): Promise<void> {
 
   try {
     if (isBrowserBinaryPresent()) return; // re-check under lock
-    await runFetch();
-    if (isBrowserBinaryPresent()) {
-      logger.info('[ensure-browser] Camoufox browser is ready.');
-    } else {
-      logger.warn('[ensure-browser] camoufox fetch reported success but no binary was found.');
+    // Guarded: a legacy flat camoufox-js install is moved aside first (the fetch, and the
+    // launcher itself, delete a non-empty dir without the compat flag), and is removed
+    // only after the new build verifies. A failed fetch restores it. See camoufox-layout.ts.
+    const result = await provisionBrowser(getCamoufoxBinaryPath(), runFetch, (m) => logger.info(`[ensure-browser] ${m}`), currentPin());
+    if (result.status === 'failed') {
+      throw new Error(result.reason ?? 'camoufox fetch failed');
     }
+    logger.info('[ensure-browser] Camoufox browser is ready.');
   } finally {
     if (haveLock) {
       try {
@@ -198,7 +236,7 @@ async function provision(): Promise<void> {
  * Idempotent and concurrency-safe: concurrent callers share a single fetch, and
  * a cross-process lock prevents two processes from fetching at the same time.
  * Best-effort — if the fetch fails, the subsequent browser launch surfaces the
- * existing actionable "run npx camoufox-js fetch" error rather than this throwing.
+ * existing actionable "run npx camoufox fetch" error rather than this throwing.
  */
 export function ensureBrowserInstalled(): Promise<void> {
   if (!inFlight) {

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -158,55 +158,51 @@ export async function initBrowser(): Promise<void> {
         logToDebugFile('INFO', `[Worker-${workerId}] Initializing browser instance...`);
 
         // ---------------------------------------------------------------------
-        // Pinned browser stack — DO NOT caret-bump these three in isolation.
-        // They are one coupled set (see package.json + docs/ARCHITECTURE.md).
-        // Each was a real fresh-install/runtime break that our lockfile masked
-        // (consumers install without our lockfile, so a floating range resolves
-        // to a newer, broken version):
+        // Pinned browser stack — DO NOT caret-bump these in isolation. They are one
+        // coupled set (see package.json + docs/ARCHITECTURE.md). Each was a real
+        // fresh-install/runtime break that our lockfile masked (consumers install
+        // without our lockfile, so a floating range resolves to a newer, broken
+        // version):
         //
-        //   • camoufox-js  ^0.12.0 (2026-08-30 refresh) → the 0.10.x pin survived
-        //     two refresh cycles on three since-lapsed blockers (Windows assets
-        //     returned in v152.0.4-beta.26, 2026-07-16; impit's `only-allow pnpm`
-        //     guard existed only in 0.13.1/0.14.0; and better-sqlite3 13's changed
-        //     install model). The refresh accepted better-sqlite3 13. MEASURED on
-        //     13.0.3: `prebuilds/` for all eight platform/arch pairs ship INSIDE
-        //     the tarball and load at runtime via node-gyp-build — no install
-        //     script, no toolchain needed on modern npm (≥11.19 skips the
-        //     injected `node-gyp rebuild` that a binding.gyp without an install
-        //     script triggers; it was that needless rebuild, not a missing
-        //     binary, that broke toolchain-less Windows installs). A future bump
-        //     must still check better-sqlite3 first, not camoufox.
-        //   • playwright-core  1.60.0 (exact) → playwright-core and the camoufox
-        //     binary are one matched Juggler-protocol pair. 1.61 added a
-        //     viewport.isMobile field the build REJECTS ("property
-        //     viewport.isMobile not described in this scheme"), failing every
-        //     launch below. Corroborated upstream: camoufox-js 0.12.0 declares
-        //     `peerDependencies: playwright-core "<1.61.0"`, the same bound this
-        //     pin has held by hand.
-        //   • impit  0.14.4 (exact, a direct dep though only camoufox-js uses it;
-        //     2026-08-30 refresh from 0.13.0) → kept EXACT because it is the only
-        //     way to force a version for consumers: npm `overrides` don't
-        //     propagate to installers of a published package, and impit's
-        //     mid-stream `only-allow pnpm` guard incident (0.13.1/0.14.0) showed
-        //     floating ranges breaking fresh global installs our lockfile masks.
+        //   • @camoufox/camoufox  ^0.5.7 (replaced camoufox-js 0.12.0, 2026-10-05).
+        //     The launcher ships `dist/data-files/browser-pin.json` naming the ONE
+        //     browser build it was released and tested with (v156.0.1-beta.34 at
+        //     0.5.7), and by default fetches and launches exactly that build. The
+        //     old camoufox-js had no such pairing: it took "the newest non-prerelease
+        //     GitHub release", so when upstream flagged v156.0.1-beta.34
+        //     non-prerelease on 2026-10-03 every fresh install got a browser whose
+        //     schema dropped `navigator.product` and every launch threw
+        //     `UnknownProperty` (CI went red on all three OSes; machines with an
+        //     older cached browser never noticed). A new pi-research release that
+        //     bumps this dependency therefore moves the browser deliberately, with
+        //     the launcher it was verified against, and no longer on upstream's
+        //     release timing.
+        //   • playwright-core  1.62.1 (exact) → playwright-core and the Camoufox
+        //     build are one matched Juggler-protocol pair. @camoufox/camoufox
+        //     declares `peerDependencies: playwright-core "<1.63"` and mirrors it
+        //     from its Python twin ("every Playwright minor is free to change
+        //     Juggler, so the ceiling is bumped deliberately"). 1.61 added a
+        //     viewport.isMobile field the OLD build rejected; the paired v156 build
+        //     was verified under 1.62.x. Re-verify a REAL launch before raising it.
+        //   • impit  0.14.5 (exact, a direct dep though only the launcher uses it):
+        //     kept EXACT because it is the only way to force a version for
+        //     consumers — npm `overrides` don't propagate to installers of a
+        //     published package, and impit's mid-stream `only-allow pnpm` guard
+        //     incident (0.13.1/0.14.0) showed floating ranges breaking fresh global
+        //     installs our lockfile masks.
         //
-        // What is NOT pinned: the browser BINARY. `camoufox-js fetch` takes no
-        // version argument — it walks the GitHub releases newest-first and takes
-        // the first non-prerelease one with an asset for this OS/arch. A consumer
-        // therefore gets whatever camoufox published most recently, whichever
-        // camoufox-js they have, and a future release could break launches with no
-        // change on our side. Both the current newest (v152.0.4-beta.28, Firefox
-        // 152) and the older v135.0.1-beta.24 an existing cache may hold were
-        // verified launching and driving under playwright-core 1.60.0.
+        // better-sqlite3 is gone: it was camoufox-js's dependency, not ours, and
+        // @camoufox/camoufox has none. The one native module left on the launch
+        // path is impit, probed by probeBrowserNativeDeps().
         //
         // Always re-verify a REAL headless run after touching any of this: the
         // unit and integration suites mock the browser, so a Juggler mismatch
         // reaches production with a fully green suite.
         let CamoufoxModule: any;
         try {
-          CamoufoxModule = await import('camoufox-js');
+          CamoufoxModule = await import('@camoufox/camoufox');
         } catch (e: unknown) {
-          throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
+          throw new Error(`[Worker] @camoufox/camoufox not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
         }
 
         const { Camoufox } = CamoufoxModule;
@@ -344,7 +340,7 @@ export async function initBrowser(): Promise<void> {
     } catch (e: unknown) {
       // Close any partially-launched browser to avoid orphaning the process.
       // Promise.resolve() wraps the call safely: in headless:'virtual' mode,
-      // camoufox-js browser.close() is synchronous (returns void), not a Promise.
+      // camoufox browser.close() is synchronous (returns void), not a Promise.
       if (browser && typeof browser.close === 'function') {
         Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during failed init: ${err.message}`));
       }
@@ -352,7 +348,7 @@ export async function initBrowser(): Promise<void> {
       const msg = e instanceof Error ? e.message : String(e);
 
       if (msg.includes('Camoufox is not installed') || msg.includes('Version information not found')) {
-        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
+        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox fetch' to install them.`, { cause: e });
       }
 
       // camoufox headless: "virtual" spawns Xvfb on Linux when no DISPLAY is set.

package/src/infrastructure/browser/thread-worker.mjs

@@ -107,7 +107,7 @@ var init_log_utils = __esm({
       "gi"
     );
     KNOWN_TOKEN_PATTERN = new RegExp(
-      `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+      `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
       "g"
     );
     JWT_PATTERN = new RegExp(
@@ -1761,9 +1761,9 @@ function formatErrorWithCause(err) {
 function isPrivateIp(ip) {
   const parts = ip.split(".").map(Number);
   if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
-  const [a, b] = parts;
+  const [a, b, c] = parts;
   if (a === void 0 || b === void 0) return false;
-  return a === 0 || a === 10 || a === 127 || a === 100 && b >= 64 && b <= 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a >= 224;
+  return a === 0 || a === 10 || a === 127 || a === 100 && b >= 64 && b <= 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 198 && (b === 18 || b === 19) || a === 192 && b === 0 && (c === 0 || c === 2) || a === 198 && b === 51 && c === 100 || a === 203 && b === 0 && c === 113 || a >= 224;
 }
 function isPrivateIpv6(ip) {
   const normalized = ip.toLowerCase().replace(/^\[|\]$/g, "");
@@ -3038,9 +3038,9 @@ async function initBrowser() {
         logToDebugFile3("INFO", `[Worker-${workerId3}] Initializing browser instance...`);
         let CamoufoxModule;
         try {
-          CamoufoxModule = await import("camoufox-js");
+          CamoufoxModule = await import("@camoufox/camoufox");
         } catch (e) {
-          throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
+          throw new Error(`[Worker] @camoufox/camoufox not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
         }
         const { Camoufox } = CamoufoxModule;
         const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
@@ -3146,7 +3146,7 @@ async function initBrowser() {
       browser = null;
       const msg = e instanceof Error ? e.message : String(e);
       if (msg.includes("Camoufox is not installed") || msg.includes("Version information not found")) {
-        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
+        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox fetch' to install them.`, { cause: e });
       }
       const errName = e instanceof Error ? e.constructor?.name ?? "" : "";
       if (errName === "CannotFindXvfb" || errName === "CannotExecuteXvfb" || msg.includes("Xvfb") || msg.includes("virtual display")) {

package/src/infrastructure/knowledge-store-service.ts

@@ -214,12 +214,12 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
             // liveOwnerStaleThreshold (120s default) that actually governs when a
             // provably-alive holder's lock may be physically reclaimed. This lock
             // protects a post-run FTS rebuild plus optimize with no fixed duration; if
-            // that holder has a stretch of non-yielding synchronous work (a single
-            // large better-sqlite3 call blocks the event loop for its full duration,
-            // during which the heartbeat literally cannot fire) longer than 20s, a
-            // waiting caller would abandon and throw "no sign of progress from its
-            // holder" against a holder that was never actually wedged — up to 100s
-            // before the lock would even become reclaim-eligible. Raised to match
+            // that holder has a stretch of non-yielding synchronous work longer than
+            // 20s (a single large native call — the FTS rebuild or optimize — blocks the
+            // event loop for its full duration, during which the heartbeat literally
+            // cannot fire), a waiting caller would abandon and throw "no sign of
+            // progress from its holder" against a holder that was never actually
+            // wedged — up to 100s before the lock would even become reclaim-eligible. Raised to match
             // liveOwnerStaleThreshold's own default so a live holder is given the same
             // grace period here that the reclaim logic itself already grants it.
             lockTimeout: 120000,

package/src/orchestration/researcher-executor.ts

@@ -60,6 +60,36 @@ export function isUnretriableResearcherError(error: unknown): boolean {
     || message.includes('insufficient_quota');
 }
 
+/**
+ * OpenRouter's *in-flight budget* 402 — a different condition from a spent balance, and a
+ * transient one: the request is refused because too many of THIS account's requests are
+ * still open, and the provider states how long the window needs in `Retry-After`. It must
+ * stay retriable (the condition clears by itself), but the ordinary 1-2s exponential
+ * backoff cannot outlast a 120s settle window: the run burns its three attempts, fails,
+ * and the round then degrades to a raw-compilation synthesis. Matched on the provider's
+ * wording rather than a bare `402`, for the same reason as the unretriable gate above.
+ */
+export function isInFlightBudgetError(error: unknown): boolean {
+  const lower = (error instanceof Error ? error.message : String(error)).toLowerCase();
+  return lower.includes('in_flight_budget_exhausted')
+    || lower.includes('would exceed your available credits given your current in-flight requests');
+}
+
+/**
+ * The `Retry-After` a provider error asked for, in ms, clamped to [5s, 60s], or null when
+ * the error carries none. Read out of the message because the researcher retry loop sees a
+ * serialized provider error, not the raw Response headers: pi's `after_provider_response`
+ * hook (src/index.ts) is the only place with headers, and it does not feed this loop.
+ */
+export function parseRetryAfterMs(error: unknown): number | null {
+  const message = error instanceof Error ? error.message : String(error);
+  const m = /["']?retry-after["']?\s*[:=]\s*["']?(\d{1,4})/i.exec(message);
+  if (!m) return null;
+  const seconds = Number(m[1]);
+  if (!Number.isFinite(seconds) || seconds <= 0) return null;
+  return Math.min(Math.max(seconds, 5), 60) * 1000;
+}
+
 /**
  * Run a single researcher with retries
  */
@@ -214,6 +244,10 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
     .replace('{{digest_section}}', () => RESEARCHER_DIGEST_SECTION)
     .trim();
 
+  // Set when the previous attempt failed on the provider's in-flight budget: the next
+  // backoff uses the provider's Retry-After instead of the small exponential step.
+  let inFlightBackoffMs: number | null = null;
+
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     // Launch boundary: never build a session for a run that is already cancelled
     // or fast-stopped. The abort sentinel keeps the wrapper's clean-cancel
@@ -225,7 +259,11 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
       return;
     }
     if (attempt > 1) {
-      const delay = Math.min(1000 * Math.pow(2, attempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
+      // A provider-declared in-flight window overrides the exponential step for exactly
+      // one attempt: the condition cannot clear in 1-2s, and the provider already said
+      // how long it needs (isInFlightBudgetError / parseRetryAfterMs).
+      const delay = inFlightBackoffMs ?? Math.min(1000 * Math.pow(2, attempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
+      inFlightBackoffMs = null;
       logger.warn(`[ResearcherExecutor] Researcher ${id} retry ${attempt - 1}/${config.RESEARCHER_MAX_RETRIES} after ${delay}ms`);
       observer?.onResearcherProgress?.(id, 'retry');
       // Abortable: a cancel during this backoff must wake the sleep immediately —
@@ -594,6 +632,14 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
         break;
       }
 
+      // Transient provider condition that needs a real wait, not a retry loop: honor the
+      // Retry-After it returned (clamped) for the next attempt instead of 1-2s.
+      if (isInFlightBudgetError(err) && attempt < maxAttempts) {
+        inFlightBackoffMs = parseRetryAfterMs(err) ?? 15000;
+        metrics.increment('researcher_inflight_budget_backoff_total', 1, { mode: 'deep', complexity: String(complexity) });
+        logger.warn(`[ResearcherExecutor] Researcher ${id} hit the provider's in-flight budget; backing off ${inFlightBackoffMs}ms (provider Retry-After, else 15s) before attempt ${attempt + 1}`);
+      }
+
       if (attempt < maxAttempts) {
         logger.warn(`[ResearcherExecutor] Researcher ${id} attempt ${attempt} failed: ${errMsg}; will retry`);
       } else {

package/src/tui/research-health.ts

@@ -69,7 +69,7 @@ export function formatHealthError(raw: string): string {
   if (raw.includes('Xvfb') || raw.includes('xvfb') || raw.includes('virtual display') || raw.includes('display server') || raw.includes('DISPLAY not set')) {
     return 'No display server found on Linux. Install Xvfb for TTY/Wayland use: sudo apt install xvfb';
   } else if (raw.includes('not found') || raw.includes('not installed') || raw.includes('binaries')) {
-    return 'Browser engine not installed. Run `npx camoufox-js fetch` to install it.';
+    return 'Browser engine not installed. Run `npx camoufox fetch` to install it.';
   } else if (raw.includes('Timeout') || raw.includes('timeout') || raw.includes('timed out')) {
     // The readiness probes check browser/state/knowledge availability — they do NOT perform a
     // web search, so a timeout here is NOT the search engine blocking us. It is almost always

package/src/utils/log-utils.ts

@@ -142,7 +142,7 @@ const AUTH_SCHEME_PATTERN = new RegExp(
 );
 // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
 const KNOWN_TOKEN_PATTERN = new RegExp(
-  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
   'g',
 );
 // JSON Web Tokens — header.payload.signature, each a base64url segment.

package/src/web-research/browser-search.ts

@@ -323,20 +323,18 @@ export async function performSearch(
 
         // A missing native addon is a BROKEN INSTALL, and the generic advice below
         // actively misleads: it sends the reader to check their network and their
-        // machine load when neither is involved and neither can be acted on. npm 12
-        // defaults `allowScripts` to off, so a plain install no longer builds
-        // camoufox-js's better-sqlite3 dependency and every browser worker dies
-        // identically. Say that instead, and say what fixes it.
+        // machine load when neither is involved and neither can be acted on. A
+        // dropped platform package (npm/cli#4828) or an install that skipped a native
+        // dependency's binding makes every browser worker die identically. Say that
+        // instead, and say what fixes it.
         if (isNativeBindingError(sampleWorkerError)) {
             metrics.increment('browser_search_total_failures_total', 1, { cause: 'native_binding' });
             throw new Error(
                 `Search completely failed: ${reason}, all with a missing native module. ` +
-                `This is an incomplete install, not a network problem: better-sqlite3 13 ` +
-                `ships prebuilt bindings for every supported platform and loads them at ` +
-                `runtime, so a missing binding means an old or broken npm install. ` +
-                `Upgrade npm to ≥12 (npm install -g npm@12) and reinstall this package — ` +
-                `modern npm skips the needless recompile that fails on toolchain-less ` +
-                `machines, and the prebuilt binding then loads as-is.` +
+                `This is an incomplete install, not a network problem: the browser launcher's ` +
+                `native dependency (impit) ships a prebuilt binding per platform as a separate ` +
+                `package, and npm sometimes drops it (npm/cli#4828). ` +
+                `Reinstall this package (npm install, or pi install again) to restore it.` +
                 (sampleWorkerError ? ` Last worker error: ${sampleWorkerError}` : '')
             );
         }

package/src/web-research/pdf-extract-worker.mjs

@@ -221,7 +221,7 @@ var AUTH_SCHEME_PATTERN = new RegExp(
   "gi"
 );
 var KNOWN_TOKEN_PATTERN = new RegExp(
-  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
   "g"
 );
 var JWT_PATTERN = new RegExp(

package/src/web-research/scraper-utils.ts

@@ -382,18 +382,27 @@ export function formatErrorWithCause(err: unknown): string {
 function isPrivateIp(ip: string): boolean {
   const parts = ip.split('.').map(Number);
   if (parts.length !== 4 || parts.some(p => isNaN(p))) return false;
-  const [a, b] = parts;
+  const [a, b, c] = parts;
   if (a === undefined || b === undefined) return false;
   // 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10 (RFC 6598 CGNAT / shared address space,
   // routinely routed to internal infra by cloud/hosting providers), 127.0.0.0/8,
   // 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4 (multicast),
-  // 240.0.0.0/4 (reserved)
+  // 240.0.0.0/4 (reserved).
+  // Also blocked: 198.18.0.0/15 (RFC 2544 benchmarking, sometimes routed by
+  // broken tooling), 192.0.0.0/24 (RFC 6890 IETF protocol assignments) and the
+  // three documentation ranges 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24
+  // (TEST-NET-1/2/3, RFC 5737). None is a legitimate scrape target, and a
+  // mis-issued NAT or a hostile page's SSRF probe can name them.
   return (
     a === 0 || a === 10 || a === 127 ||
     (a === 100 && b >= 64 && b <= 127) ||
     (a === 169 && b === 254) ||
     (a === 172 && b >= 16 && b <= 31) ||
     (a === 192 && b === 168) ||
+    (a === 198 && (b === 18 || b === 19)) ||
+    (a === 192 && b === 0 && (c === 0 || c === 2)) ||
+    (a === 198 && b === 51 && c === 100) ||
+    (a === 203 && b === 0 && c === 113) ||
     a >= 224
   );
 }

package/src/web-research/web-scraper.ts

@@ -41,7 +41,7 @@ let playwrightAvailable: boolean = false;
 let markdownConverterPromise: Promise<(html: string) => Promise<string>> | null = null;
 
 export function initScraperDependencies(): void {
-  playwrightAvailable = checkModule('playwright-core') && checkModule('camoufox-js');
+  playwrightAvailable = checkModule('playwright-core') && checkModule('@camoufox/camoufox');
 }
 initScraperDependencies();