@lincoln504/pi-research 1.5.2 → 1.6.0

package/scripts/cleanup.cjs

@@ -28,9 +28,14 @@ const projectRoot = path.join(__dirname, '..');
  * coding-agent harnesses, so `npm uninstall` is symmetric with the in-app
  * installer (the /research-config "Install Skill in Coding Agents" action).
  * Reads the manifest written by the installer and removes ONLY entries we still
- * own (a symlink pointing into a pi-research skills dir, or a copy carrying our
- * package marker). Foreign directories are never touched. Best-effort; never
+ * own (a symlink pointing into a pi-research skill source dir, or a copy carrying
+ * our package marker). Foreign directories are never touched. Best-effort; never
  * throws (preuninstall must not fail the uninstall).
+ *
+ * The ownership regex below duplicates src/skill-install/skill-installer.ts's:
+ * preuninstall runs as a plain CJS script with no access to the built module.
+ * Both accept the current `agent-skill/` source dir AND the pre-1.5.3 `skills/`
+ * one, so a link created by an older version is still recognised and removed.
  */
 function removeInstalledSkills() {
   const manifestPath = path.join(os.homedir(), '.pi', 'research', 'installed-skills.json');
@@ -58,7 +63,7 @@ function removeInstalledSkills() {
     try {
       if (lst.isSymbolicLink()) {
         const dest = path.resolve(path.dirname(p), readlinkSync(p));
-        owned = /[/\\]pi-research[/\\]skills[/\\]pi-research$/.test(dest);
+        owned = /[/\\]pi-research[/\\](?:agent-skill|skills)[/\\]pi-research$/.test(dest);
       } else {
         const md = readFileSync(path.join(p, 'SKILL.md'), 'utf-8');
         owned = md.includes('@lincoln504/pi-research');

package/scripts/prepare.cjs

@@ -10,7 +10,7 @@
  * Artifacts built:
  *   - dist/thread-worker.mjs       (browser worker, via build:worker)
  *   - dist/cli.mjs (+ dist/prompts, dist/webgpu-probe.mjs)  (pi-research CLI binary, via build:cli)
- *   - skills/pi-research/scripts/run.mjs  (skill launcher, via build:skill)
+ *   - agent-skill/pi-research/scripts/run.mjs  (skill launcher, via build:skill)
  */
 
 'use strict';

package/src/cli.ts

@@ -2,7 +2,7 @@
  * pi-research — standalone CLI
  *
  * A thin, dependency-free command line over the pi-research programmatic SDK
- * (src/sdk.ts). It lets any tool — Claude, Cursor, a shell script, CI — run
+ * (src/sdk.ts). It lets any tool — Claude, Codex, a shell script, CI — run
  * multi-agent web research without the pi CLI or TUI.
  *
  *   pi-research research "<query>" [--depth N] [--model provider/id]

package/src/infrastructure/browser/browser-client.ts

@@ -16,6 +16,7 @@ import { errorTracker } from '../../utils/error-tracker.ts';
 import { redactSecrets } from '../../utils/log-utils.ts';
 import { Utf8Body } from '../../utils/http-body.ts';
 import { getClientAgent } from './client-agent.ts';
+import { isTransientSocketError } from './browser-error-utils.ts';
 import { getHealthCheckBudgetMs } from './config.ts';
 import type { NodeError } from '../../types/index.ts';
 
@@ -281,12 +282,29 @@ export class BrowserClient implements IScheduler {
                     errorType = 'unknown';
                 }
                 const error = new Error(errorMsg);
-                logger.error(`[BrowserClient] Request to http://127.0.0.1:${this.port}${path} failed:`, errorMsg);
-                errorTracker.trackError(error, {
-                    component: 'browser-manager',
-                    operation,
-                    errorType,
-                });
+                // This is one ATTEMPT against the leader, not an outcome. runBrowserTask
+                // retries every transient socket failure — a leader handover (ECONNREFUSED
+                // against the departing leader's port) is the common case and recovers a
+                // few hundred milliseconds later. Logging ERROR and tracking here made every
+                // recovered handover look like a fault twice over: once in the ERROR log,
+                // and once more in the error tracker on top of the caller's own richer
+                // record (taskType + errorType) on the retry path. Observed 5 of 5 times in
+                // a real run log, each followed immediately by a successful recovery.
+                //
+                // A failure the caller will NOT retry still reports at ERROR here, and a
+                // retryable one that exhausts its budget still surfaces upstream with URL
+                // context (see web-scraper's isBenignScrapeFailure branch), so nothing that
+                // actually costs the user a result becomes invisible.
+                if (isTransientSocketError(error)) {
+                    logger.debug(`[BrowserClient] Transient failure talking to http://127.0.0.1:${this.port}${path} (caller will retry): ${errorMsg}`);
+                } else {
+                    logger.error(`[BrowserClient] Request to http://127.0.0.1:${this.port}${path} failed:`, errorMsg);
+                    errorTracker.trackError(error, {
+                        component: 'browser-manager',
+                        operation,
+                        errorType,
+                    });
+                }
                 reject(error);
             });
             req.write(JSON.stringify(data));

package/src/infrastructure/browser/browser-error-utils.ts

@@ -124,6 +124,59 @@ export function isPoolShutdownError(error: unknown): boolean {
     );
 }
 
+/** Playwright navigation failures whose cause is at the remote end — see below. */
+const BENIGN_NAVIGATION_FAILURE =
+  /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
+
+/**
+ * True when a scrape failure is an EXPECTED per-URL outcome rather than an engine
+ * fault. Scraping the open web routinely hits bot-protection interstitials, thin
+ * "stub" pages, HTTP 4xx/5xx, slow/unresponsive remotes, and (on quit) pool-drain
+ * races. None of these is a pi-research bug: the URL is already accounted for as
+ * "not scraped" (scrape_results_total{outcome=total_failure}) and surfaced to the
+ * user that way, and no operator action follows. Callers use this to log such
+ * failures at debug instead of error — keeping the ERROR log (and, via
+ * logger.error's error-tracker re-track, the diagnostic error count) focused on
+ * genuine faults. The messages matched here are produced by validateContent
+ * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
+ * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
+ * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
+ * exceeds the deliberate 100MB policy cap, is routine on the open web and
+ * just as non-actionable as a stub page or a 404; the 25MB HTML policy caps
+ * (`HTML response too large` from the fetch layer, `Browser HTML too large`
+ * from the browser layer) are the same deliberate-cap class; the remaining
+ * cases delegate to the shared browser-layer predicates. Deliberately does NOT
+ * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
+ * failure) — that is an infrastructure fault affecting every URL, not a
+ * routine per-PDF outcome, and must still surface at ERROR.
+ *
+ * Navigation failures reported by Playwright (`page.goto: …`) are the same class
+ * once — and only once — the reason is a network, TLS or timeout condition at the
+ * remote end: an expired/self-signed certificate (`SEC_ERROR_UNKNOWN_ISSUER` on a
+ * live university PDF host), a DNS or connection error (`net::ERR_…`,
+ * `NS_ERROR_…`), or a site too slow to answer inside the scrape budget. These were
+ * logged at ERROR by both the leader and the scraper. `page.goto` failures with any
+ * OTHER reason — notably `Target closed`, which is how a crashed or force-closed
+ * browser surfaces — are deliberately excluded and still report as faults.
+ */
+export function isBenignScrapeFailure(error: unknown): boolean {
+  const msg = error instanceof Error ? error.message : String(error);
+  return (
+    msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+    msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
+    msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
+    msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
+    msg.startsWith('Unsupported download:') ||             // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+    BENIGN_NAVIGATION_FAILURE.test(msg) ||                 // remote TLS/DNS/timeout — the URL could not be loaded
+    msg.startsWith('HTML response too large') ||          // fetch-layer 25MB policy cap — same class as the PDF cap
+    msg.startsWith('Browser HTML too large') ||           // browser-layer 25MB policy cap
+    /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
+    isCloudflareBlockError(error) ||
+    isTaskTimeoutError(error) ||
+    isPoolShutdownError(error)
+  );
+}
+
 // Session-scoped circuit breaker storage to support concurrent research runs
 const sessionCircuitBreakers = new Map<string, CircuitBreaker>();
 // Hard cap so a cleanup path that ever skips clearSessionCircuitBreaker (e.g. an earlier

package/src/infrastructure/browser/browser-server.ts

@@ -3,7 +3,7 @@ import * as crypto from 'node:crypto';
 import { logger } from '../../logger.ts';
 import { Utf8Body } from '../../utils/http-body.ts';
 import type { SearchResult } from '../../web-research/types.ts';
-import { isCloudflareBlockError, isPoolShutdownError } from './browser-error-utils.ts';
+import { isCloudflareBlockError, isPoolShutdownError, isBenignScrapeFailure } from './browser-error-utils.ts';
 
 export interface BrowserServerOptions {
     onSearch: (query: string, signal?: AbortSignal) => Promise<SearchResult[]>;
@@ -204,6 +204,14 @@ export class BrowserServer {
                             // Pool destroyed mid-request during quit/SIGTERM teardown — expected,
                             // not a server fault. DEBUG so a normal shutdown doesn't emit ERRORs.
                             logger.debug('[BrowserServer] Request abandoned during shutdown:', error instanceof Error ? error.message : String(error));
+                        } else if (isBenignScrapeFailure(error)) {
+                            // Per-URL outcomes the CLIENT side already classifies as routine —
+                            // an HTTP 4xx/5xx, a navigation timeout, an oversized PDF, a blocked
+                            // fetch. The leader saw the same failure first and logged it at ERROR
+                            // regardless, so every one of those outcomes still produced exactly
+                            // one false ERROR per URL despite the client's classification. Same
+                            // predicate as web-scraper.ts, so the two layers agree.
+                            logger.debug('[BrowserServer] Expected per-URL scrape outcome:', error instanceof Error ? error.message.split('\n')[0] : String(error));
                         } else {
                             logger.error('[BrowserServer] Error handling request:', error);
                         }

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -368,7 +368,10 @@ export async function initBrowser(): Promise<void> {
  * rather than left to leak.
  */
 export async function acquireTaskContext(): Promise<any> {
-  const contextPromise = browser.newContext();
+  // acceptDownloads is explicit, not defaulted: executeScrapeTask recovers PDFs that a
+  // server sends as an attachment by reading the resulting Download, and that object
+  // only exists when downloads are accepted.
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId: NodeJS.Timeout | undefined;
   let contextRaceLost = false;

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -6,7 +6,7 @@
 
 let workerId: string = '';
 
-import { appendFileSync } from 'node:fs';
+import { appendFileSync, promises as fsp } from 'node:fs';
 import { redactSecrets } from '../../utils/log-utils.ts';
 import { safeUnref } from '../../utils/safe-unref.ts';
 import { MAX_HTML_SIZE, MAX_PDF_SIZE } from '../../web-research/scraper-types.ts';
@@ -405,6 +405,146 @@ function seedSubresourceVerdict(validatedUrl: string): void {
   }
 }
 
+/**
+ * True when Firefox has discarded a response body it once held, so `response.body()`
+ * can never succeed for that response again. Distinct from every other body failure —
+ * a closed target, an aborted task — which must keep their original error.
+ */
+function isEvictedBodyError(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+
+/** Upper bound on the PDF re-fetch below. A 14MB PDF measured ~20s on a normal link. */
+const PDF_REFETCH_TIMEOUT_MS = 90_000;
+
+/**
+ * Re-request a PDF whose navigation body Firefox has already discarded.
+ *
+ * Uses the browser context's own request API rather than a bare fetch so the retry
+ * carries the same cookies, headers, proxy and TLS profile the navigation used — the
+ * reason this URL reached the stealth-browser fallback in the first place is usually
+ * that a plain fetch does not get served.
+ *
+ * SSRF: this is the one request in the scrape path that does NOT pass through
+ * `page.route` or fire `page.on('response')`, so neither the request-time DNS check
+ * nor the serverAddr rebinding backstop covers it. Three things stand in for them:
+ * the URL is re-validated here with the same DNS-aware check the route guard uses;
+ * it is the FINAL navigation URL, whose actually-connected IP the caller validated
+ * via `response.serverAddr()` seconds earlier; and redirects are refused outright, so
+ * the re-fetch cannot be walked to an unvalidated host. What remains is a TTL-0
+ * rebind landing inside the seconds between the navigation and this retry, on a host
+ * that already answered from a public address — strictly narrower than the
+ * subresource TOCTOU window the request-time check already accepts.
+ */
+async function refetchPdfBody(
+  page: any,
+  url: string,
+  validateForWorker: (u: string) => Promise<unknown>,
+  signal?: AbortSignal,
+): Promise<Buffer> {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error('Aborted');
+
+  // Raced against the task signal, not merely checked before starting. Playwright's
+  // request API takes no AbortSignal, and closing the page does not cancel it — so
+  // without this race a slow retry would hold the poolifier worker slot for up to
+  // PDF_REFETCH_TIMEOUT_MS past the deadline that was supposed to free it. The request
+  // itself is left to settle and be discarded; what matters is that the slot is not
+  // pinned waiting for it.
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS,
+  });
+  requestPromise.catch(() => {});
+  const response = await (signal
+    ? Promise.race([
+        requestPromise,
+        new Promise<never>((_, reject) => {
+          if (signal.aborted) return reject(new Error('Aborted'));
+          signal.addEventListener('abort', () => reject(new Error('Aborted')), { once: true });
+        }),
+      ])
+    : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  // A 3xx here means the server wants to send us somewhere maxRedirects:0 refused to
+  // follow — report it as the HTTP outcome it is rather than reading a redirect body.
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h: any) => String(h.name).toLowerCase() === 'content-length')?.value ?? ''), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+
+  const buffer: Buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile('DEBUG', `[Worker-${workerId}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+
+/**
+ * Finish a scrape from a Download the navigation turned into.
+ *
+ * A server that sends a PDF with `Content-Disposition: attachment` never produces a
+ * document for Playwright to inspect — Firefox hands the response to the download
+ * manager and `page.goto()` rejects with "Download is starting". The bytes have
+ * already arrived at that point, so the URL is recoverable; before this path existed
+ * every such source (vendor datasheets, standards-body PDFs, several journal hosts)
+ * was reported as an outright scrape failure.
+ *
+ * The SSRF guarantees are the ones the normal PDF branch relies on: the download's
+ * response passed through the same `page.on('response')` inspection, so settling
+ * `pendingAddrChecks` and re-checking the poison gate before returning bytes is
+ * equivalent here.
+ */
+async function readPdfDownload(
+  download: any,
+  url: string,
+  page: any,
+  pendingAddrChecks: Array<Promise<void>>,
+  getPoisonedError: () => Error | null,
+): Promise<{ contentType: string; bufferB64: string; jitter: number }> {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+
+  // Only PDFs are usable as binary content downstream; anything else (zip, csv, xlsx)
+  // is a routine unsupported-target outcome for this URL, not a fault.
+  const suggested = String(download.suggestedFilename?.() ?? '');
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf =
+    suggested.toLowerCase().endsWith('.pdf') || downloadUrl.toLowerCase().split('?')[0]!.endsWith('.pdf');
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+
+  const filePath = await download.path();
+  if (!filePath) throw new Error('Fetch blocked: download produced no readable file');
+
+  // Same 100MB policy cap the PDF branch enforces, checked from the file size so the
+  // bytes are never read into the worker at all when the file is over the limit.
+  const { size } = await fsp.stat(filePath);
+  if (size > MAX_PDF_SIZE) {
+    const sizeMB = Math.round(size / 1024 / 1024);
+    throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+  }
+
+  const buffer = await fsp.readFile(filePath);
+
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+
+  // Delete before the page closes: the context's download directory is removed on
+  // context close, but a worker's context is reused across tasks, so an undeleted
+  // file would accumulate for the life of the worker.
+  await download.delete?.().catch(() => {});
+  await page.close();
+  logToDebugFile('DEBUG', `[Worker-${workerId}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: 'application/pdf', bufferB64: buffer.toString('base64'), jitter: 0 };
+}
+
 /**
  * Execute a scrape task
  */
@@ -446,6 +586,14 @@ export async function executeScrapeTask(
   // page.close() produces.
   let oversizedPdfError: Error | null = null;
 
+  // Set when the navigation turns into a download instead of a document. A server that
+  // sends the PDF with Content-Disposition: attachment makes Firefox hand the response
+  // to the download manager, and page.goto() then rejects with "Download is starting" —
+  // the PDF branch below was never reached and the document was lost even though the
+  // bytes had already arrived. Captured here so the goto catch can finish the scrape
+  // from the downloaded file instead of failing the URL.
+  let pendingDownload: any = null;
+
   try {
     logToDebugFile('DEBUG', `[Worker-${workerId}] Starting scrape for: ${url}`);
 
@@ -589,8 +737,32 @@ export async function executeScrapeTask(
       }
     });
 
+    // A navigation the server marks as an attachment never becomes a document: Firefox
+    // routes it to the download manager and goto() rejects. Record the Download object
+    // so the goto catch below can read the bytes that already arrived.
+    page.on('download', (d: any) => { pendingDownload ??= d; });
+
     // High-fidelity wait: try domcontentloaded first for speed
-    const response = await page.goto(url, { waitUntil: 'domcontentloaded' });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' });
+    } catch (navErr: unknown) {
+      // Only the download case is recoverable here; everything else (timeout, DNS,
+      // 'Target closed' from an abort/poison) must keep its original error.
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      // The 'download' event and the goto rejection are not ordered relative to each
+      // other, so give the event a brief moment to land before giving up on it.
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent('download', { timeout: 5000 }).catch(() => null),
+          new Promise(r => setTimeout(r, 5000)),
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
 
     // DNS-rebinding defense-in-depth. The page.route guard's validateForWorker resolves the
     // hostname independently of the browser's own connect-time resolution, so a TTL-0 rebind can
@@ -647,7 +819,18 @@ export async function executeScrapeTask(
       // decided this and be in the middle of closing the page — skip the (redundant,
       // about-to-fail) download attempt rather than race it.
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer: Buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        // Firefox drops large navigation response bodies from its juggler-side cache,
+        // after which response.body() can only ever fail. Measured against a 13.9MB
+        // TU Delft lecture PDF: the body is unavailable whether it is requested at
+        // headers-received time or after the load settles, so this is not a race that
+        // asking earlier can win — the bytes are simply no longer held for us.
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       // ...then the actual bytes: the header is advisory and absent on chunked
       // responses, so the byte count is the check that actually bounds memory.
       // Checked BEFORE toString('base64') doubles the allocation.

package/src/infrastructure/browser/thread-worker.mjs

@@ -1516,11 +1516,25 @@ function isPoolShutdownError(error) {
   // same transient swap the "being reset" string covers, not a hard fault.
   err.message.includes("Cannot execute a task on not started pool"));
 }
-var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
+function isBenignScrapeFailure(error) {
+  const msg = error instanceof Error ? error.message : String(error);
+  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
+  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
+  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
+  msg.startsWith("Unsupported download:") || // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+  BENIGN_NAVIGATION_FAILURE.test(msg) || // remote TLS/DNS/timeout — the URL could not be loaded
+  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
+  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
+  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
+  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
+}
+var BENIGN_NAVIGATION_FAILURE, DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({
   "src/infrastructure/browser/browser-error-utils.ts"() {
     "use strict";
     init_circuit_breaker();
+    BENIGN_NAVIGATION_FAILURE = /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
     DEFAULT_BREAKER_CONFIG = {
       // Raised from 3→8: parallel search bursts of 20 queries frequently produce
       // simultaneous timeouts on slow/CF-protected sites. A threshold of 3 caused
@@ -1565,17 +1579,6 @@ import * as dnsCb from "node:dns";
 import * as net from "node:net";
 import crypto from "node:crypto";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-function isBenignScrapeFailure(error) {
-  const msg = error instanceof Error ? error.message : String(error);
-  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
-  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
-  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
-  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
-  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
-  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
-  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
-}
 function getRandomUserAgent() {
   try {
     const index = crypto.randomInt(0, USER_AGENTS.length);
@@ -2014,7 +2017,7 @@ import { platform as platform2 } from "node:os";
 
 // src/infrastructure/browser/thread-worker-messaging.ts
 init_log_utils();
-import { appendFileSync as appendFileSync2 } from "node:fs";
+import { appendFileSync as appendFileSync2, promises as fsp } from "node:fs";
 
 // src/utils/safe-unref.ts
 function safeUnref(timer) {
@@ -2286,6 +2289,66 @@ function seedSubresourceVerdict(validatedUrl) {
   } catch {
   }
 }
+function isEvictedBodyError(err) {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+var PDF_REFETCH_TIMEOUT_MS = 9e4;
+async function refetchPdfBody(page, url, validateForWorker, signal) {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error("Aborted");
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS
+  });
+  requestPromise.catch(() => {
+  });
+  const response = await (signal ? Promise.race([
+    requestPromise,
+    new Promise((_, reject) => {
+      if (signal.aborted) return reject(new Error("Aborted"));
+      signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
+    })
+  ]) : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h) => String(h.name).toLowerCase() === "content-length")?.value ?? ""), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+  const buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+async function readPdfDownload(download, url, page, pendingAddrChecks, getPoisonedError) {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+  const suggested = String(download.suggestedFilename?.() ?? "");
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf = suggested.toLowerCase().endsWith(".pdf") || downloadUrl.toLowerCase().split("?")[0].endsWith(".pdf");
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+  const filePath = await download.path();
+  if (!filePath) throw new Error("Fetch blocked: download produced no readable file");
+  const { size } = await fsp.stat(filePath);
+  if (size > MAX_PDF_SIZE) {
+    const sizeMB = Math.round(size / 1024 / 1024);
+    throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+  }
+  const buffer = await fsp.readFile(filePath);
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+  await download.delete?.().catch(() => {
+  });
+  await page.close();
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: "application/pdf", bufferB64: buffer.toString("base64"), jitter: 0 };
+}
 async function executeScrapeTask(_context, url, signal) {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
@@ -2302,6 +2365,7 @@ async function executeScrapeTask(_context, url, signal) {
   let poisonedError = null;
   const pendingAddrChecks = [];
   let oversizedPdfError = null;
+  let pendingDownload = null;
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
     const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2368,7 +2432,25 @@ async function executeScrapeTask(_context, url, signal) {
       } catch {
       }
     });
-    const response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    page.on("download", (d) => {
+      pendingDownload ??= d;
+    });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    } catch (navErr) {
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent("download", { timeout: 5e3 }).catch(() => null),
+          new Promise((r) => setTimeout(r, 5e3))
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
     const serverAddr = await response?.serverAddr().catch(() => null);
     if (serverAddr?.ipAddress) {
       const ip = serverAddr.ipAddress;
@@ -2386,7 +2468,13 @@ async function executeScrapeTask(_context, url, signal) {
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       if (buffer.byteLength > MAX_PDF_SIZE) {
         const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
@@ -2749,7 +2837,7 @@ function logToDebugFile3(level, ...args) {
         return String(arg);
       }).join(" "))
     };
-    import("node:fs/promises").then((fsp) => fsp.appendFile(logFile, `${JSON.stringify(entry)}
+    import("node:fs/promises").then((fsp2) => fsp2.appendFile(logFile, `${JSON.stringify(entry)}
 `)).catch(() => {
     });
   } catch {
@@ -2888,7 +2976,7 @@ async function initBrowser() {
   return initPromise;
 }
 async function acquireTaskContext() {
-  const contextPromise = browser.newContext();
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId;
   let contextRaceLost = false;

package/src/knowledge/store-operations.ts

@@ -13,6 +13,13 @@ import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
 import { StoreDocument } from '../core/interfaces/knowledge-interfaces.ts';
 
+/**
+ * What a single add attempt actually did. Returned rather than logged/counted here
+ * so the caller — which owns the retry loop — records exactly one outcome per
+ * logical addDocuments call.
+ */
+export type AddDocumentsOutcome = 'written' | 'empty' | 'ignored_closing';
+
 /**
  * Guard against a broken embedder silently writing corrupt vectors: a
  * wrong-count, empty, ragged, or NaN/Inf-poisoned result would otherwise land
@@ -69,12 +76,11 @@ export async function addDocumentsToStore(
   isClosing: () => boolean,
   workspace: string,
   isGlobal: boolean
-): Promise<void> {
-  if (docs.length === 0) return;
+): Promise<AddDocumentsOutcome> {
+  if (docs.length === 0) return 'empty';
   if (isClosing()) {
     logger.warn('[store] Ignoring addDocuments during close');
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
-    return;
+    return 'ignored_closing';
   }
 
   const startTime = Date.now();
@@ -98,14 +104,19 @@ export async function addDocumentsToStore(
     await table.add(data);
     const duration = Date.now() - startTime;
     metrics.observe('knowledge_store_add_documents_duration_ms', duration);
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'success' });
     metrics.increment('knowledge_store_chunks_added_total', docs.length);
     logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [workspace=${workspace}, global=${isGlobal}]`);
+    return 'written';
   } catch (err) {
     const duration = Date.now() - startTime;
     metrics.observe('knowledge_store_add_documents_duration_ms', duration, { status: 'error' });
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
-    logger.error('[store] Failed to add documents:', err);
+    // Attempt-level failure only. The caller retries embedder-unreachable and Lance
+    // commit-conflict failures and usually succeeds, so logging ERROR and counting a
+    // terminal outcome here reported lost writes that were not lost, and double-counted
+    // the batch in knowledge_store_add_documents_total (once error, once success).
+    // Outcome accounting belongs to KnowledgeStore.addDocuments, which is the only
+    // layer that knows whether the write was ultimately abandoned.
+    logger.debug(`[store] add attempt failed after ${duration}ms: ${err instanceof Error ? err.message : String(err)}`);
     throw err;
   }
 }