@lincoln504/pi-research 1.7.7 → 1.7.9

package/src/infrastructure/browser/ensure-browser.ts

@@ -13,28 +13,28 @@
  *
  * Non-regression guarantee for the other install paths: when the browser is
  * already present — which it always is after the pi-extension / plain-npm
- * postinstall fetch — this is a cheap `existsSync` check that returns
- * immediately and changes nothing. It only ever does work when the binary is
- * genuinely absent.
+ * postinstall fetch — this is a cheap layout check that returns
+ * immediately and changes nothing. It only ever does work when the paired
+ * browser build is genuinely absent (or a legacy flat install needs migrating).
  */
 
 import { spawn } from 'node:child_process';
 import { createRequire } from 'node:module';
 import { createHash } from 'node:crypto';
-import { existsSync, readdirSync, statSync, fstatSync, openSync, closeSync, rmSync } from 'node:fs';
+import { existsSync, fstatSync, openSync, closeSync, rmSync, readFileSync } from 'node:fs';
 import { join, dirname } from 'node:path';
-import { tmpdir } from 'node:os';
+import { tmpdir, platform, homedir } from 'node:os';
 import { logger } from '../../logger.ts';
 import { getCamoufoxBinaryPath } from './config.ts';
+import { resolveInstall, isUsable, provisionBrowser, loadPin, findPackageRoot, type BrowserPin } from './camoufox-layout.ts';
 
-/** Bound the ~500MB camoufox-js 0.12 browser download (measured 484MB; the
- *  0.10 pin was ~100MB and the old bound assumed that) so a stalled network
- *  fails eventually instead of hanging forever, while a slow-but-working link
- *  can actually finish. */
+/** Bound the ~1.3GB Camoufox browser download (the paired build is ~1.29 GB) so a stalled
+ *  network fails eventually instead of hanging forever, while a slow-but-working link can
+ *  actually finish. */
 const FETCH_TIMEOUT_MS = 45 * 60 * 1000;
 /** A lock older than this is considered stale (crashed mid-fetch) and may be stolen.
  *  Must exceed FETCH_TIMEOUT_MS: stealing a LIVE fetcher's lock mid-download would
- *  start a second 500MB download beside the first. */
+ *  start a second download beside the first. */
 const STALE_LOCK_MS = 55 * 60 * 1000;
 /** When another process holds the fetch lock, poll this long for the browser to appear.
  *  Spans a full FETCH_TIMEOUT_MS fetch, so a legitimate slow peer is waited out rather
@@ -45,55 +45,68 @@ const POLL_INTERVAL_MS = 2000;
 /** In-process dedupe: concurrent pool inits / scrapers share one fetch. */
 let inFlight: Promise<void> | null = null;
 
+/** The paired browser build this installed @camoufox/camoufox was released with. */
+function currentPin(): BrowserPin | null {
+  const root = findPackageRoot(import.meta.url);
+  return root ? loadPin(root) : null;
+}
+
 /**
- * True when a camoufox browser binary is actually present (cache dir exists and
- * contains at least one version subdirectory) — mirrors the check in setup.cjs.
+ * True when a usable Camoufox browser is installed: the new layout's compat flag plus
+ * the build this launcher is paired with (any build when the user chose one explicitly).
+ * A bare "directory exists" is NOT enough: a legacy flat camoufox-js install is a
+ * non-empty directory too, and the launcher deletes it on first use.
  */
 export function isBrowserBinaryPresent(): boolean {
   try {
-    const cacheDir = getCamoufoxBinaryPath();
-    if (!existsSync(cacheDir)) return false;
-    return readdirSync(cacheDir).some((entry) => {
-      try {
-        return statSync(join(cacheDir, entry)).isDirectory();
-      } catch {
-        return false;
-      }
-    });
+    return isUsable(getCamoufoxBinaryPath(), currentPin());
   } catch {
     return false;
   }
 }
 
-/** Resolve the camoufox-js CLI bin, mirroring scripts/setup.cjs (handles hoisting). */
-function resolveCamoufoxBin(): string | null {
+/** Resolve the `camoufox` CLI entry (JS file) of @camoufox/camoufox, mirroring scripts/setup.cjs. */
+function resolveCamoufoxCli(): string | null {
   try {
     const require = createRequire(import.meta.url);
-    const pkgJson = require.resolve('camoufox-js/package.json');
-    const bin = join(dirname(pkgJson), '..', '.bin', process.platform === 'win32' ? 'camoufox-js.cmd' : 'camoufox-js');
-    if (existsSync(bin)) return bin;
+    const pkgJson = require.resolve('@camoufox/camoufox/package.json');
+    const pkg = JSON.parse(readFileSync(pkgJson, 'utf8')) as { bin?: string | Record<string, string> };
+    const rel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.['camoufox'];
+    if (rel) {
+      const entry = join(dirname(pkgJson), rel);
+      if (existsSync(entry)) return entry;
+    }
   } catch {
-    /* fall through to npx */
+    /* fall through */
   }
   return null;
 }
 
 const delay = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
 
-/** Run `camoufox-js fetch`, resolving on exit 0 and rejecting otherwise / on timeout. */
-function runFetch(): Promise<void> {
-  return new Promise<void>((resolve, reject) => {
-    const bin = resolveCamoufoxBin();
-    const [cmd, args] = bin ? [bin, ['fetch']] : ['npx', ['camoufox-js', 'fetch']];
-    logger.info(`[ensure-browser] Camoufox not found; fetching the browser (this runs once, ~500MB): ${cmd} ${args.join(' ')}`);
-
-    // Windows needs shell:true to run the .cmd shim (Node CVE-2024-27980 fix makes a
-    // shell-less spawn of .cmd throw EINVAL), but shell:true performs NO quoting, so a
-    // spaced install path (C:\Users\John Smith\…) must be quoted explicitly. The args
-    // here are fixed literals ('fetch'), never user input, so quoting the command alone
-    // is sufficient and this stays injection-free.
-    const useShell = process.platform === 'win32';
-    const child = spawn(useShell ? `"${cmd}"` : cmd, args, { stdio: 'inherit', shell: useShell });
+/**
+ * Run `camoufox fetch`, resolving true on exit 0, false on any other exit, rejecting on
+ * spawn error / timeout. Runs the CLI's JS entry with process.execPath and an argv array:
+ * no shell, so neither a spaced Windows path nor a metacharacter in an install directory
+ * can be reinterpreted.
+ */
+function runFetch(): Promise<boolean> {
+  return new Promise<boolean>((resolve, reject) => {
+    const cli = resolveCamoufoxCli();
+    if (!cli) {
+      reject(new Error('@camoufox/camoufox is not installed (cannot find its CLI); run npm install'));
+      return;
+    }
+    // The launcher reads only XDG_CACHE_HOME / LOCALAPPDATA: hand it the override derived
+    // from the user's CAMOUFOX_INSTALL_DIR / PLAYWRIGHT_BROWSERS_PATH so the fetch lands
+    // where the lookup (getCamoufoxBinaryPath) expects it.
+    const install = resolveInstall(process.env, platform(), homedir());
+    const env: NodeJS.ProcessEnv = { ...process.env, ...install.overrides };
+    delete env['CAMOUFOX_INSTALL_DIR'];
+    delete env['PLAYWRIGHT_BROWSERS_PATH'];
+    logger.info(`[ensure-browser] Camoufox not found; fetching the browser (this runs once, ~1.3GB): ${process.execPath} ${cli} fetch`);
+
+    const child = spawn(process.execPath, [cli, 'fetch'], { stdio: 'inherit', env });
     const timer = setTimeout(() => {
       child.kill('SIGKILL');
       reject(new Error(`camoufox fetch timed out after ${FETCH_TIMEOUT_MS}ms`));
@@ -105,8 +118,7 @@ function runFetch(): Promise<void> {
     });
     child.on('exit', (code) => {
       clearTimeout(timer);
-      if (code === 0) resolve();
-      else reject(new Error(`camoufox fetch exited with code ${code}`));
+      resolve(code === 0);
     });
   });
 }
@@ -175,12 +187,14 @@ async function provision(): Promise<void> {
 
   try {
     if (isBrowserBinaryPresent()) return; // re-check under lock
-    await runFetch();
-    if (isBrowserBinaryPresent()) {
-      logger.info('[ensure-browser] Camoufox browser is ready.');
-    } else {
-      logger.warn('[ensure-browser] camoufox fetch reported success but no binary was found.');
+    // Guarded: a legacy flat camoufox-js install is moved aside first (the fetch, and the
+    // launcher itself, delete a non-empty dir without the compat flag), and is removed
+    // only after the new build verifies. A failed fetch restores it. See camoufox-layout.ts.
+    const result = await provisionBrowser(getCamoufoxBinaryPath(), runFetch, (m) => logger.info(`[ensure-browser] ${m}`), currentPin());
+    if (result.status === 'failed') {
+      throw new Error(result.reason ?? 'camoufox fetch failed');
     }
+    logger.info('[ensure-browser] Camoufox browser is ready.');
   } finally {
     if (haveLock) {
       try {
@@ -198,7 +212,7 @@ async function provision(): Promise<void> {
  * Idempotent and concurrency-safe: concurrent callers share a single fetch, and
  * a cross-process lock prevents two processes from fetching at the same time.
  * Best-effort — if the fetch fails, the subsequent browser launch surfaces the
- * existing actionable "run npx camoufox-js fetch" error rather than this throwing.
+ * existing actionable "run npx camoufox fetch" error rather than this throwing.
  */
 export function ensureBrowserInstalled(): Promise<void> {
   if (!inFlight) {

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -158,55 +158,51 @@ export async function initBrowser(): Promise<void> {
         logToDebugFile('INFO', `[Worker-${workerId}] Initializing browser instance...`);
 
         // ---------------------------------------------------------------------
-        // Pinned browser stack — DO NOT caret-bump these three in isolation.
-        // They are one coupled set (see package.json + docs/ARCHITECTURE.md).
-        // Each was a real fresh-install/runtime break that our lockfile masked
-        // (consumers install without our lockfile, so a floating range resolves
-        // to a newer, broken version):
+        // Pinned browser stack — DO NOT caret-bump these in isolation. They are one
+        // coupled set (see package.json + docs/ARCHITECTURE.md). Each was a real
+        // fresh-install/runtime break that our lockfile masked (consumers install
+        // without our lockfile, so a floating range resolves to a newer, broken
+        // version):
         //
-        //   • camoufox-js  ^0.12.0 (2026-08-30 refresh) → the 0.10.x pin survived
-        //     two refresh cycles on three since-lapsed blockers (Windows assets
-        //     returned in v152.0.4-beta.26, 2026-07-16; impit's `only-allow pnpm`
-        //     guard existed only in 0.13.1/0.14.0; and better-sqlite3 13's changed
-        //     install model). The refresh accepted better-sqlite3 13. MEASURED on
-        //     13.0.3: `prebuilds/` for all eight platform/arch pairs ship INSIDE
-        //     the tarball and load at runtime via node-gyp-build — no install
-        //     script, no toolchain needed on modern npm (≥11.19 skips the
-        //     injected `node-gyp rebuild` that a binding.gyp without an install
-        //     script triggers; it was that needless rebuild, not a missing
-        //     binary, that broke toolchain-less Windows installs). A future bump
-        //     must still check better-sqlite3 first, not camoufox.
-        //   • playwright-core  1.60.0 (exact) → playwright-core and the camoufox
-        //     binary are one matched Juggler-protocol pair. 1.61 added a
-        //     viewport.isMobile field the build REJECTS ("property
-        //     viewport.isMobile not described in this scheme"), failing every
-        //     launch below. Corroborated upstream: camoufox-js 0.12.0 declares
-        //     `peerDependencies: playwright-core "<1.61.0"`, the same bound this
-        //     pin has held by hand.
-        //   • impit  0.14.4 (exact, a direct dep though only camoufox-js uses it;
-        //     2026-08-30 refresh from 0.13.0) → kept EXACT because it is the only
-        //     way to force a version for consumers: npm `overrides` don't
-        //     propagate to installers of a published package, and impit's
-        //     mid-stream `only-allow pnpm` guard incident (0.13.1/0.14.0) showed
-        //     floating ranges breaking fresh global installs our lockfile masks.
+        //   • @camoufox/camoufox  ^0.5.7 (replaced camoufox-js 0.12.0, 2026-10-05).
+        //     The launcher ships `dist/data-files/browser-pin.json` naming the ONE
+        //     browser build it was released and tested with (v156.0.1-beta.34 at
+        //     0.5.7), and by default fetches and launches exactly that build. The
+        //     old camoufox-js had no such pairing: it took "the newest non-prerelease
+        //     GitHub release", so when upstream flagged v156.0.1-beta.34
+        //     non-prerelease on 2026-10-03 every fresh install got a browser whose
+        //     schema dropped `navigator.product` and every launch threw
+        //     `UnknownProperty` (CI went red on all three OSes; machines with an
+        //     older cached browser never noticed). A new pi-research release that
+        //     bumps this dependency therefore moves the browser deliberately, with
+        //     the launcher it was verified against, and no longer on upstream's
+        //     release timing.
+        //   • playwright-core  1.62.1 (exact) → playwright-core and the Camoufox
+        //     build are one matched Juggler-protocol pair. @camoufox/camoufox
+        //     declares `peerDependencies: playwright-core "<1.63"` and mirrors it
+        //     from its Python twin ("every Playwright minor is free to change
+        //     Juggler, so the ceiling is bumped deliberately"). 1.61 added a
+        //     viewport.isMobile field the OLD build rejected; the paired v156 build
+        //     was verified under 1.62.x. Re-verify a REAL launch before raising it.
+        //   • impit  0.14.5 (exact, a direct dep though only the launcher uses it):
+        //     kept EXACT because it is the only way to force a version for
+        //     consumers — npm `overrides` don't propagate to installers of a
+        //     published package, and impit's mid-stream `only-allow pnpm` guard
+        //     incident (0.13.1/0.14.0) showed floating ranges breaking fresh global
+        //     installs our lockfile masks.
         //
-        // What is NOT pinned: the browser BINARY. `camoufox-js fetch` takes no
-        // version argument — it walks the GitHub releases newest-first and takes
-        // the first non-prerelease one with an asset for this OS/arch. A consumer
-        // therefore gets whatever camoufox published most recently, whichever
-        // camoufox-js they have, and a future release could break launches with no
-        // change on our side. Both the current newest (v152.0.4-beta.28, Firefox
-        // 152) and the older v135.0.1-beta.24 an existing cache may hold were
-        // verified launching and driving under playwright-core 1.60.0.
+        // better-sqlite3 is gone: it was camoufox-js's dependency, not ours, and
+        // @camoufox/camoufox has none. The one native module left on the launch
+        // path is impit, probed by probeBrowserNativeDeps().
         //
         // Always re-verify a REAL headless run after touching any of this: the
         // unit and integration suites mock the browser, so a Juggler mismatch
         // reaches production with a fully green suite.
         let CamoufoxModule: any;
         try {
-          CamoufoxModule = await import('camoufox-js');
+          CamoufoxModule = await import('@camoufox/camoufox');
         } catch (e: unknown) {
-          throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
+          throw new Error(`[Worker] @camoufox/camoufox not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
         }
 
         const { Camoufox } = CamoufoxModule;
@@ -344,7 +340,7 @@ export async function initBrowser(): Promise<void> {
     } catch (e: unknown) {
       // Close any partially-launched browser to avoid orphaning the process.
       // Promise.resolve() wraps the call safely: in headless:'virtual' mode,
-      // camoufox-js browser.close() is synchronous (returns void), not a Promise.
+      // camoufox browser.close() is synchronous (returns void), not a Promise.
       if (browser && typeof browser.close === 'function') {
         Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during failed init: ${err.message}`));
       }
@@ -352,7 +348,7 @@ export async function initBrowser(): Promise<void> {
       const msg = e instanceof Error ? e.message : String(e);
 
       if (msg.includes('Camoufox is not installed') || msg.includes('Version information not found')) {
-        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
+        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox fetch' to install them.`, { cause: e });
       }
 
       // camoufox headless: "virtual" spawns Xvfb on Linux when no DISPLAY is set.

package/src/infrastructure/browser/thread-worker.mjs

@@ -60,7 +60,12 @@ function redactSecrets(message) {
   out = out.replace(BASIC_AUTH_PATTERN, "Basic [REDACTED]");
   out = out.replace(BEARER_AUTH_PATTERN, "Bearer [REDACTED]");
   out = out.replace(COOKIE_HEADER_PATTERN, (_m, key, sep) => `${key}${sep}[REDACTED]`);
-  out = out.replace(SENSITIVE_KV_PATTERN, (_m, key, sep) => `${key}${sep}[REDACTED]`);
+  out = out.replace(AUTH_SCHEME_PATTERN, (_m, key, sep, scheme) => `${key}${sep}${scheme} [REDACTED]`);
+  out = out.replace(SENSITIVE_KV_PATTERN, (_m, key, sep, val) => {
+    const q = val[0] === '"' || val[0] === "'" ? val[0] : "";
+    const closed = q !== "" && val.length > 1 && val.endsWith(q);
+    return `${key}${sep}${q}[REDACTED]${closed ? q : ""}`;
+  });
   out = out.replace(KNOWN_TOKEN_PATTERN, "[REDACTED]");
   out = out.replace(PROVIDER_TOKEN_PATTERN, "[REDACTED]");
   out = out.replace(LONG_HEX_SECRET_PATTERN, "[REDACTED]");
@@ -79,7 +84,7 @@ function redactSecrets(message) {
 function stripTerminalEscapes(message) {
   return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
 }
-var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, B_LEFT, B_RIGHT, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
+var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, B_LEFT, B_RIGHT, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, AUTH_SCHEME_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
 var init_log_utils = __esm({
   "src/utils/log-utils.ts"() {
     "use strict";
@@ -94,11 +99,15 @@ var init_log_utils = __esm({
       "gi"
     );
     SENSITIVE_KV_PATTERN = new RegExp(
-      `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
+      `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|token|key)${B_RIGHT}(["']?\\s*[:=]\\s*)("[^"\\r\\n]{0,1024}"|'[^'\\r\\n]{0,1024}'|["']?[^\\s"',&)]+)`,
+      "gi"
+    );
+    AUTH_SCHEME_PATTERN = new RegExp(
+      `${B_LEFT}(authorization)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([A-Za-z][A-Za-z0-9_-]{0,31})[ \\t]+[^\\s"',&)]+`,
       "gi"
     );
     KNOWN_TOKEN_PATTERN = new RegExp(
-      `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+      `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
       "g"
     );
     JWT_PATTERN = new RegExp(
@@ -1561,6 +1570,7 @@ var init_browser_error_utils = __esm({
 var scraper_utils_exports = {};
 __export(scraper_utils_exports, {
   cleanConvertedMarkdown: () => cleanConvertedMarkdown,
+  collapseBlankLines: () => collapseBlankLines,
   createJsMarkdownConverter: () => createJsMarkdownConverter,
   createNativeMarkdownConverter: () => createNativeMarkdownConverter,
   disposeSsrfSafeFetcher: () => disposeSsrfSafeFetcher,
@@ -1751,9 +1761,9 @@ function formatErrorWithCause(err) {
 function isPrivateIp(ip) {
   const parts = ip.split(".").map(Number);
   if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
-  const [a, b] = parts;
+  const [a, b, c] = parts;
   if (a === void 0 || b === void 0) return false;
-  return a === 0 || a === 10 || a === 127 || a === 100 && b >= 64 && b <= 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a >= 224;
+  return a === 0 || a === 10 || a === 127 || a === 100 && b >= 64 && b <= 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 198 && (b === 18 || b === 19) || a === 192 && b === 0 && (c === 0 || c === 2) || a === 198 && b === 51 && c === 100 || a === 203 && b === 0 && c === 113 || a >= 224;
 }
 function isPrivateIpv6(ip) {
   const normalized = ip.toLowerCase().replace(/^\[|\]$/g, "");
@@ -1815,10 +1825,28 @@ ${kept.join("\n")}
 ${rest}` : rest;
 }
 function cleanConvertedMarkdown(markdown) {
-  return stripEmptyFragmentLinks(stripImageLinks(trimFrontMatter(markdown))).replace(/\n{3,}/g, "\n\n").trim();
+  return collapseBlankLines(stripEmptyFragmentLinks(stripImageLinks(trimFrontMatter(markdown)))).trim();
+}
+function collapseBlankLines(markdown) {
+  if (!markdown.includes("\n\n\n")) return markdown;
+  const out = [];
+  let fence = null;
+  for (const line of markdown.split("\n")) {
+    const m = CODE_FENCE.exec(line);
+    if (fence === null) {
+      if (m) fence = m[1];
+    } else if (m && m[1][0] === fence[0] && m[1].length >= fence.length && line.slice(m[0].length).trim() === "") {
+      fence = null;
+    }
+    if (fence === null && line === "" && out.length > 0 && out[out.length - 1] === "") continue;
+    out.push(line);
+  }
+  return out.join("\n");
 }
 function stripImageLinks(markdown) {
-  return markdown.replace(MARKDOWN_IMAGE_PATTERN, "").replace(IMAGE_LINK_PATTERN, "$1").replace(/\n{3,}/g, "\n\n").trim();
+  return collapseBlankLines(
+    markdown.replace(MARKDOWN_IMAGE_PATTERN, "").replace(IMAGE_LINK_PATTERN, "$1")
+  ).trim();
 }
 function createNativeMarkdownConverter(nativeModule) {
   return async (html) => {
@@ -1880,7 +1908,7 @@ function validateContent(html, markdown, url) {
     throw error;
   }
 }
-var cachedSsrfFetcher, FRONT_MATTER_SLOTS, MAX_FRONT_MATTER_CHARS, FRONT_MATTER_KEY;
+var cachedSsrfFetcher, FRONT_MATTER_SLOTS, MAX_FRONT_MATTER_CHARS, FRONT_MATTER_KEY, CODE_FENCE;
 var init_scraper_utils = __esm({
   "src/web-research/scraper-utils.ts"() {
     "use strict";
@@ -1907,6 +1935,7 @@ var init_scraper_utils = __esm({
     ];
     MAX_FRONT_MATTER_CHARS = 2e4;
     FRONT_MATTER_KEY = /^([\w.:-]+):(?:\s|$)/;
+    CODE_FENCE = /^ {0,3}(`{3,}|~{3,})/;
   }
 });
 
@@ -3009,9 +3038,9 @@ async function initBrowser() {
         logToDebugFile3("INFO", `[Worker-${workerId3}] Initializing browser instance...`);
         let CamoufoxModule;
         try {
-          CamoufoxModule = await import("camoufox-js");
+          CamoufoxModule = await import("@camoufox/camoufox");
         } catch (e) {
-          throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
+          throw new Error(`[Worker] @camoufox/camoufox not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
         }
         const { Camoufox } = CamoufoxModule;
         const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
@@ -3117,7 +3146,7 @@ async function initBrowser() {
       browser = null;
       const msg = e instanceof Error ? e.message : String(e);
       if (msg.includes("Camoufox is not installed") || msg.includes("Version information not found")) {
-        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
+        throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox fetch' to install them.`, { cause: e });
       }
       const errName = e instanceof Error ? e.constructor?.name ?? "" : "";
       if (errName === "CannotFindXvfb" || errName === "CannotExecuteXvfb" || msg.includes("Xvfb") || msg.includes("virtual display")) {

package/src/infrastructure/knowledge-store-service.ts

@@ -214,12 +214,12 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
             // liveOwnerStaleThreshold (120s default) that actually governs when a
             // provably-alive holder's lock may be physically reclaimed. This lock
             // protects a post-run FTS rebuild plus optimize with no fixed duration; if
-            // that holder has a stretch of non-yielding synchronous work (a single
-            // large better-sqlite3 call blocks the event loop for its full duration,
-            // during which the heartbeat literally cannot fire) longer than 20s, a
-            // waiting caller would abandon and throw "no sign of progress from its
-            // holder" against a holder that was never actually wedged — up to 100s
-            // before the lock would even become reclaim-eligible. Raised to match
+            // that holder has a stretch of non-yielding synchronous work longer than
+            // 20s (a single large native call — the FTS rebuild or optimize — blocks the
+            // event loop for its full duration, during which the heartbeat literally
+            // cannot fire), a waiting caller would abandon and throw "no sign of
+            // progress from its holder" against a holder that was never actually
+            // wedged — up to 100s before the lock would even become reclaim-eligible. Raised to match
             // liveOwnerStaleThreshold's own default so a live holder is given the same
             // grace period here that the reclaim logic itself already grants it.
             lockTimeout: 120000,

package/src/knowledge/store.ts

@@ -279,7 +279,7 @@ export class KnowledgeStore implements IKnowledgeStore {
 
     try {
       if (!fs.existsSync(this.options.dbDir)) {
-        fs.mkdirSync(this.options.dbDir, { recursive: true });
+        fs.mkdirSync(this.options.dbDir, { recursive: true, mode: 0o700 });
       }
       this.db = await (await getLancedb()).connect(this.options.dbDir);
 

package/src/orchestration/researcher-executor.ts

@@ -60,6 +60,36 @@ export function isUnretriableResearcherError(error: unknown): boolean {
     || message.includes('insufficient_quota');
 }
 
+/**
+ * OpenRouter's *in-flight budget* 402 — a different condition from a spent balance, and a
+ * transient one: the request is refused because too many of THIS account's requests are
+ * still open, and the provider states how long the window needs in `Retry-After`. It must
+ * stay retriable (the condition clears by itself), but the ordinary 1-2s exponential
+ * backoff cannot outlast a 120s settle window: the run burns its three attempts, fails,
+ * and the round then degrades to a raw-compilation synthesis. Matched on the provider's
+ * wording rather than a bare `402`, for the same reason as the unretriable gate above.
+ */
+export function isInFlightBudgetError(error: unknown): boolean {
+  const lower = (error instanceof Error ? error.message : String(error)).toLowerCase();
+  return lower.includes('in_flight_budget_exhausted')
+    || lower.includes('would exceed your available credits given your current in-flight requests');
+}
+
+/**
+ * The `Retry-After` a provider error asked for, in ms, clamped to [5s, 60s], or null when
+ * the error carries none. Read out of the message because the researcher retry loop sees a
+ * serialized provider error, not the raw Response headers: pi's `after_provider_response`
+ * hook (src/index.ts) is the only place with headers, and it does not feed this loop.
+ */
+export function parseRetryAfterMs(error: unknown): number | null {
+  const message = error instanceof Error ? error.message : String(error);
+  const m = /["']?retry-after["']?\s*[:=]\s*["']?(\d{1,4})/i.exec(message);
+  if (!m) return null;
+  const seconds = Number(m[1]);
+  if (!Number.isFinite(seconds) || seconds <= 0) return null;
+  return Math.min(Math.max(seconds, 5), 60) * 1000;
+}
+
 /**
  * Run a single researcher with retries
  */
@@ -214,6 +244,10 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
     .replace('{{digest_section}}', () => RESEARCHER_DIGEST_SECTION)
     .trim();
 
+  // Set when the previous attempt failed on the provider's in-flight budget: the next
+  // backoff uses the provider's Retry-After instead of the small exponential step.
+  let inFlightBackoffMs: number | null = null;
+
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     // Launch boundary: never build a session for a run that is already cancelled
     // or fast-stopped. The abort sentinel keeps the wrapper's clean-cancel
@@ -225,7 +259,11 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
       return;
     }
     if (attempt > 1) {
-      const delay = Math.min(1000 * Math.pow(2, attempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
+      // A provider-declared in-flight window overrides the exponential step for exactly
+      // one attempt: the condition cannot clear in 1-2s, and the provider already said
+      // how long it needs (isInFlightBudgetError / parseRetryAfterMs).
+      const delay = inFlightBackoffMs ?? Math.min(1000 * Math.pow(2, attempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
+      inFlightBackoffMs = null;
       logger.warn(`[ResearcherExecutor] Researcher ${id} retry ${attempt - 1}/${config.RESEARCHER_MAX_RETRIES} after ${delay}ms`);
       observer?.onResearcherProgress?.(id, 'retry');
       // Abortable: a cancel during this backoff must wake the sleep immediately —
@@ -594,6 +632,14 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
         break;
       }
 
+      // Transient provider condition that needs a real wait, not a retry loop: honor the
+      // Retry-After it returned (clamped) for the next attempt instead of 1-2s.
+      if (isInFlightBudgetError(err) && attempt < maxAttempts) {
+        inFlightBackoffMs = parseRetryAfterMs(err) ?? 15000;
+        metrics.increment('researcher_inflight_budget_backoff_total', 1, { mode: 'deep', complexity: String(complexity) });
+        logger.warn(`[ResearcherExecutor] Researcher ${id} hit the provider's in-flight budget; backing off ${inFlightBackoffMs}ms (provider Retry-After, else 15s) before attempt ${attempt + 1}`);
+      }
+
       if (attempt < maxAttempts) {
         logger.warn(`[ResearcherExecutor] Researcher ${id} attempt ${attempt} failed: ${errMsg}; will retry`);
       } else {

package/src/tui/research-health.ts

@@ -69,7 +69,7 @@ export function formatHealthError(raw: string): string {
   if (raw.includes('Xvfb') || raw.includes('xvfb') || raw.includes('virtual display') || raw.includes('display server') || raw.includes('DISPLAY not set')) {
     return 'No display server found on Linux. Install Xvfb for TTY/Wayland use: sudo apt install xvfb';
   } else if (raw.includes('not found') || raw.includes('not installed') || raw.includes('binaries')) {
-    return 'Browser engine not installed. Run `npx camoufox-js fetch` to install it.';
+    return 'Browser engine not installed. Run `npx camoufox fetch` to install it.';
   } else if (raw.includes('Timeout') || raw.includes('timeout') || raw.includes('timed out')) {
     // The readiness probes check browser/state/knowledge availability — they do NOT perform a
     // web search, so a timeout here is NOT the search engine blocking us. It is almost always

package/src/utils/log-utils.ts

@@ -126,13 +126,23 @@ const COOKIE_HEADER_PATTERN = new RegExp(
 // sort key, a map key) is the accepted trade-off — same one already made for
 // the equally generic bare `session`/`cookie` entries below.
 const SENSITIVE_KV_PATTERN = new RegExp(
-  `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}` +
-    `(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
+  `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|token|key)${B_RIGHT}` +
+    // A quoted value is consumed whole (bounded, so no backtracking blow-up): the
+    // old unquoted class stopped at the first space/comma/&, so `password="a b"`
+    // left ` b"` in the clear. An unterminated quote falls through to that class.
+    `(["']?\\s*[:=]\\s*)("[^"\\r\\n]{0,1024}"|'[^'\\r\\n]{0,1024}'|["']?[^\\s"',&)]+)`,
+  'gi',
+);
+// `Authorization: <scheme> <credential>` for schemes the Bearer/Basic patterns do
+// not know (Token, Digest, ApiKey, ...). The KV pass alone would mask only the
+// scheme word and leave the credential after the space.
+const AUTH_SCHEME_PATTERN = new RegExp(
+  `${B_LEFT}(authorization)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([A-Za-z][A-Za-z0-9_-]{0,31})[ \\t]+[^\\s"',&)]+`,
   'gi',
 );
 // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
 const KNOWN_TOKEN_PATTERN = new RegExp(
-  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|npm_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
   'g',
 );
 // JSON Web Tokens — header.payload.signature, each a base64url segment.
@@ -237,7 +247,12 @@ export function redactSecrets(message: string): string {
   // Whole-header cookie masking before the KV pass, which would otherwise stop
   // at the first whitespace and leave every subsequent pair in the clear.
   out = out.replace(COOKIE_HEADER_PATTERN, (_m, key: string, sep: string) => `${key}${sep}[REDACTED]`);
-  out = out.replace(SENSITIVE_KV_PATTERN, (_m, key: string, sep: string) => `${key}${sep}[REDACTED]`);
+  out = out.replace(AUTH_SCHEME_PATTERN, (_m, key: string, sep: string, scheme: string) => `${key}${sep}${scheme} [REDACTED]`);
+  out = out.replace(SENSITIVE_KV_PATTERN, (_m, key: string, sep: string, val: string) => {
+    const q = val[0] === '"' || val[0] === "'" ? val[0] : '';
+    const closed = q !== '' && val.length > 1 && val.endsWith(q);
+    return `${key}${sep}${q}[REDACTED]${closed ? q : ''}`;
+  });
   out = out.replace(KNOWN_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(PROVIDER_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(LONG_HEX_SECRET_PATTERN, '[REDACTED]');

package/src/utils/research-export.ts

@@ -188,7 +188,7 @@ export async function exportResearchReport(
     const filepath = join(targetDir, filename);
 
     try {
-      await fs.writeFile(filepath, result, { flag: 'wx' });
+      await fs.writeFile(filepath, result, { flag: 'wx', mode: 0o600 });
       logger.log(`[export] Research report saved to: ${filepath}`);
       return filepath;
     } catch (error) {