@lincoln504/pi-research 1.5.3 → 1.6.1

package/dist/thread-worker.mjs

@@ -1516,11 +1516,25 @@ function isPoolShutdownError(error) {
   // same transient swap the "being reset" string covers, not a hard fault.
   err.message.includes("Cannot execute a task on not started pool"));
 }
-var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
+function isBenignScrapeFailure(error) {
+  const msg = error instanceof Error ? error.message : String(error);
+  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
+  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
+  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
+  msg.startsWith("Unsupported download:") || // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+  BENIGN_NAVIGATION_FAILURE.test(msg) || // remote TLS/DNS/timeout — the URL could not be loaded
+  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
+  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
+  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
+  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
+}
+var BENIGN_NAVIGATION_FAILURE, DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({
   "src/infrastructure/browser/browser-error-utils.ts"() {
     "use strict";
     init_circuit_breaker();
+    BENIGN_NAVIGATION_FAILURE = /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
     DEFAULT_BREAKER_CONFIG = {
       // Raised from 3→8: parallel search bursts of 20 queries frequently produce
       // simultaneous timeouts on slow/CF-protected sites. A threshold of 3 caused
@@ -1565,17 +1579,6 @@ import * as dnsCb from "node:dns";
 import * as net from "node:net";
 import crypto from "node:crypto";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-function isBenignScrapeFailure(error) {
-  const msg = error instanceof Error ? error.message : String(error);
-  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
-  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
-  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
-  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
-  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
-  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
-  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
-}
 function getRandomUserAgent() {
   try {
     const index = crypto.randomInt(0, USER_AGENTS.length);
@@ -2014,7 +2017,7 @@ import { platform as platform2 } from "node:os";
 
 // src/infrastructure/browser/thread-worker-messaging.ts
 init_log_utils();
-import { appendFileSync as appendFileSync2 } from "node:fs";
+import { appendFileSync as appendFileSync2, promises as fsp } from "node:fs";
 
 // src/utils/safe-unref.ts
 function safeUnref(timer) {
@@ -2286,6 +2289,73 @@ function seedSubresourceVerdict(validatedUrl) {
   } catch {
   }
 }
+function isEvictedBodyError(err) {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+var PDF_REFETCH_TIMEOUT_MS = 9e4;
+async function refetchPdfBody(page, url, validateForWorker, signal) {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error("Aborted");
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS
+  });
+  requestPromise.catch(() => {
+  });
+  const response = await (signal ? Promise.race([
+    requestPromise,
+    new Promise((_, reject) => {
+      if (signal.aborted) return reject(new Error("Aborted"));
+      signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
+    })
+  ]) : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h) => String(h.name).toLowerCase() === "content-length")?.value ?? ""), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+  const buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+async function readPdfDownload(download, url, page, pendingAddrChecks, getPoisonedError) {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+  const suggested = String(download.suggestedFilename?.() ?? "");
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf = suggested.toLowerCase().endsWith(".pdf") || downloadUrl.toLowerCase().split("?")[0].endsWith(".pdf");
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+  const filePath = await download.path();
+  if (!filePath) throw new Error("Fetch blocked: download produced no readable file");
+  const handle = await fsp.open(filePath, "r");
+  let buffer;
+  try {
+    const { size } = await handle.stat();
+    if (size > MAX_PDF_SIZE) {
+      const sizeMB = Math.round(size / 1024 / 1024);
+      throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+    }
+    buffer = await handle.readFile();
+  } finally {
+    await handle.close().catch(() => {
+    });
+  }
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+  await download.delete?.().catch(() => {
+  });
+  await page.close();
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: "application/pdf", bufferB64: buffer.toString("base64"), jitter: 0 };
+}
 async function executeScrapeTask(_context, url, signal) {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
@@ -2302,6 +2372,7 @@ async function executeScrapeTask(_context, url, signal) {
   let poisonedError = null;
   const pendingAddrChecks = [];
   let oversizedPdfError = null;
+  let pendingDownload = null;
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
     const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2368,7 +2439,25 @@ async function executeScrapeTask(_context, url, signal) {
       } catch {
       }
     });
-    const response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    page.on("download", (d) => {
+      pendingDownload ??= d;
+    });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    } catch (navErr) {
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent("download", { timeout: 5e3 }).catch(() => null),
+          new Promise((r) => setTimeout(r, 5e3))
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
     const serverAddr = await response?.serverAddr().catch(() => null);
     if (serverAddr?.ipAddress) {
       const ip = serverAddr.ipAddress;
@@ -2386,7 +2475,13 @@ async function executeScrapeTask(_context, url, signal) {
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       if (buffer.byteLength > MAX_PDF_SIZE) {
         const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
@@ -2749,7 +2844,7 @@ function logToDebugFile3(level, ...args) {
         return String(arg);
       }).join(" "))
     };
-    import("node:fs/promises").then((fsp) => fsp.appendFile(logFile, `${JSON.stringify(entry)}
+    import("node:fs/promises").then((fsp2) => fsp2.appendFile(logFile, `${JSON.stringify(entry)}
 `)).catch(() => {
     });
   } catch {
@@ -2888,7 +2983,7 @@ async function initBrowser() {
   return initPromise;
 }
 async function acquireTaskContext() {
-  const contextPromise = browser.newContext();
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId;
   let contextRaceLost = false;

package/docs/CHANGELOG.md

@@ -5,6 +5,69 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.1] - 2026-08-23
+
+A report that contained no analysis was being delivered as a completed run, because
+the engine's own repair step padded it out. Fixing that exposed a second way into the
+same failure, and a review of the machinery around it turned up a third defect.
+
+### Fixed
+
+- **A synthesis with no analysis in it reached users looking like a finished report.** The completion guard was `if (!result.trim())` — a lone title line passes it, and `ensureCitedLinks` then appends the rebuilt source list underneath. Two runs in the retained logs did exactly that: 131 and 147 characters of prose delivered as 6.8KB and 9.9KB documents with a heading, a full citation list, and not one sentence between them, both reported as complete. The engine's own repair machinery was what disguised the failure. A synthesis is now checked for analysis by two independent signals — prose volume measured BEFORE the citation block, and whether any sentence ends at all — and the response is **additive**: the model's own text is kept, the researchers' reports are appended below it, and a notice states what happened. That is the part that matters. Because a false positive can only ever add material, never discard it, the threshold can sit well clear of real reports (800 characters, against a smallest observed legitimate report of 1056) rather than hugging the two failures out of fear of destroying a good run. Quick mode gets the structural signal only: it has no researcher reports to fall back on, so the sole available response is to flag the report — and telling a reader their correct one-sentence answer is untrustworthy is worse than missing a thin one. Its notice says the run produced no substantive write-up and that the cited sources are still accurate (`src/orchestration/deep-research-orchestrator.ts`, `src/orchestration/quick-research-orchestrator.ts`, `src/utils/text-utils.ts`, `src/constants.ts`).
+- **The same failure had a second route in, through the raw JSON envelope.** When upstream JSON parsing fails, the synthesis arrives as the whole envelope string rather than the extracted markdown, and an unwrap ran afterwards to recover it. Both completion checks therefore measured the envelope — long, well punctuated, and healthy-looking whatever it wrapped — and the unwrap then replaced it with the title or the empty string it was hiding, with nothing left to catch either. An envelope wrapping empty content produced a citation list with no document above it. The unwrap now runs first, so every check judges the report itself (`src/orchestration/deep-research-orchestrator.ts`).
+- **The structural signal assumed Latin punctuation.** It looked for `.!?` plus the CJK full stops, so a correct report written with a danda, an Arabic full stop or an Ethiopic full stop ended no sentence as far as the check was concerned. In quick mode, where that signal runs alone, every such report would have been flagged as thin (`src/utils/text-utils.ts`).
+- **A time-of-check/time-of-use gap in the attachment-served PDF path.** The 100MB cap was measured with `fsp.stat(filePath)` and the bytes then read with `fsp.readFile(filePath)` — two separate lookups of a path the browser process is still writing to, so the file that was measured need not be the file that was read (CodeQL `js/file-system-race`, alert #266). The size is now taken from an open handle and the bytes read back through that same handle, which cannot be swapped underneath, and the descriptor is released on both the accept and the over-cap reject (`src/infrastructure/browser/thread-worker-messaging.ts`).
+
+### Changed
+
+- **A quick report no longer opens by narrating itself.** Live runs delivered documents whose first line was "I have gathered extensive material… Let me now synthesize my findings into a comprehensive report." and "I have gathered sufficient material from multiple authoritative sources. I will now synthesize the full report." Quick mode has no synthesis step behind it to absorb that, so it landed as the reader's first sentence. The researcher prompt now forbids it — and, because that instruction was measurably not obeyed (it still appeared in 2 of 3 live runs after the prompt change), quick mode also removes the line. Removing text is destructive, so every condition on it is a required narrowing rather than a score: the segment must be a single line, separated from the report by a blank line, first-person AND about the act of reporting, and must leave a report behind. A report that opens as the format requires — with a topic title — matches none of them. The strip runs BEFORE the analysis check above, so a title-only report cannot pass that check on the strength of the narration's full stop, and each removal is counted in `research_report_preamble_stripped_total` (`src/prompts/researcher.md`, `src/utils/text-utils.ts`, `src/orchestration/quick-research-orchestrator.ts`).
+
+### Removed
+
+- **The GitHub Release job added in 1.6.0 is gone; tags remain the only release marker.** Cutting a Release per tag was not wanted — the tag plus `docs/CHANGELOG.md` is the intended record — so the job was removed and the one Release it created (v1.6.0) was deleted. The tag itself is untouched, and nothing about publishing changes: the release workflow is still CI-gate → tarball test → npm publish (`.github/workflows/release.yml`).
+
+### Verified
+
+- Full gates green: 2917 unit tests across 230 files, type-check over source and tests, lint, dependency-cruiser, and the integration suite (77 passed, 26 skipped for live network).
+- Live runs across all four front-ends: CLI quick and depth-1 (15.5KB and 16.5KB reports), the agent skill launcher through its installed link (11.8KB, opening straight at the report), the pi extension, and the SDK (12.1KB, clean shutdown). Every report was replayed through both guards afterwards: the one real preamble was removed, and neither analysis check fired on any of them.
+
+## [1.6.0] - 2026-08-22
+
+Driven by log forensics rather than by the test suite: every entry below was found
+by reading what the engine actually reported in production, and each fix was
+re-measured against the exact URL, error line or counter that exposed it.
+
+The starting observation was that the ERROR log had stopped meaning anything. Across
+the retained run logs, **all 26 ERROR lines were either a condition the engine had
+already recovered from or a routine per-URL outcome** — a blocked page, a slow site,
+an untrusted certificate. Not one described an unrecovered engine fault. Two classes
+account for all of them: content the scraper had already fetched and then dropped,
+and recovered conditions reported as terminal failures.
+
+### Fixed
+
+- **PDF sources the browser had already fetched were being discarded two different ways, both now recovered.** A server sending the PDF with `Content-Disposition: attachment` makes Firefox route the response to the download manager, so no document is produced and `page.goto()` rejects with "Download is starting" — the PDF branch was never reached even though the bytes had arrived. Separately, Firefox discards large navigation response bodies from its juggler-side cache, after which `response.body()` fails with "Response body … was evicted!". Both recur across the run logs, and both hit exactly the sources research most wants — vendor datasheets, standards-body documents, journal and university PDFs. The download is now read back from disk, and an evicted body is re-requested through the browser context's own request API — which carries the same cookies, headers, proxy and TLS profile as the navigation, the reason the URL reached the stealth-browser fallback at all. Verified live against the two URLs that failed: a Frontiers article PDF now yields 106k characters of text and a 13.9MB TU Delft lecture PDF 27k, both previously total failures. The re-fetch was measured before it was written: asking for the body earlier does not help, because the eviction happens when the browser drops the bytes, not when we ask for them — a first attempt built on that assumption was discarded after a direct camoufox probe disproved it (`src/infrastructure/browser/thread-worker-messaging.ts`, `src/infrastructure/browser/thread-worker-browser.ts`).
+- **A knowledge-store write that was retried and succeeded no longer reports as a failed write.** `addDocumentsToStore` logged ERROR and incremented `knowledge_store_add_documents_total{status=error}` before rethrowing — but its caller catches exactly that throw, reconnects to a fresh embedder and retries, so the write usually completed. All 8 occurrences across the retained logs are immediately followed by the reconnect WARN — 8 of 8 recovered, none lost — and every one of them counted the same batch twice (once error, once success). The Lance commit-conflict retry fed the same path, so cross-process write contention did it too. Attempt-level accounting now stays in the attempt and outcome accounting moved to `KnowledgeStore.addDocuments`, which is the only layer that knows whether a write was actually abandoned — exactly one counter increment per call, and an ERROR only when the batch is genuinely lost (`src/knowledge/store-operations.ts`, `src/knowledge/store.ts`).
+- **The browser leader logged every routine per-URL scrape outcome as a server fault.** `BrowserServer`'s handler classified Cloudflare challenges and pool-drain races but sent everything else to ERROR, including the HTTP 4xx/5xx, navigation timeouts and over-cap PDFs that the client side already classifies as expected. The leader saw each failure first, so the client's careful classification never prevented the ERROR — one false fault per affected URL. Both layers now share the same predicate (`src/infrastructure/browser/browser-server.ts`).
+- **A leader handover was reported as a terminal error by the layer that was about to retry it.** `BrowserClient` logged ERROR and recorded a tracked error for any socket failure, including the ECONNREFUSED a follower gets against a leader that has already exited. `runBrowserTask` retries that within a second: 5 of 5 such lines in the log were followed immediately by the recovery WARN, and each was counted twice in the diagnostic error report — once here without context, once by the caller with it. A failure the caller will not retry still reports at ERROR, and a retryable one that exhausts its budget still surfaces upstream with URL context (`src/infrastructure/browser/browser-client.ts`).
+- **Remote TLS, DNS and navigation-timeout failures are now classified as the per-URL outcomes they are.** `page.goto: SEC_ERROR_UNKNOWN_ISSUER` (a live university PDF host with an untrusted certificate) and `page.goto: Timeout 15000ms exceeded` reached the ERROR log and the diagnostic error count from both the leader and the scraper, though neither is actionable and the URL is already tallied as not scraped. `page.goto: Target closed` — how a crashed or force-closed browser surfaces — is deliberately excluded and still reports as a fault (`src/web-research/scraper-utils.ts`).
+
+- **The unit suite was writing its deliberate failure fixtures into the user’s real shared log.** `unit-env.ts` redirects `PI_RESEARCH_LOG_PATH` before `src/logger.ts` is evaluated, but the global logger is built lazily and resolves its path at CONSTRUCTION — so a suite whose `beforeEach` deletes every `PI_RESEARCH_*` variable (config.test.ts did) left the first logger of the run binding to the production default instead. Measured at ~7KB per run appended to `/tmp/pi-research.log`, including a `{corrupt json` registry fixture whose ERROR line reads exactly like a live failure and cost real time during this release’s own forensic pass. The setup now constructs the logger while the redirect is still set, config.test.ts keeps the redirect out of its scrub, and a test pins the guarantee; whole-suite pollution measured at 0 bytes after the fix (`test/setup/unit.ts`, `test/unit/config.test.ts`, `test/unit/logger-test-isolation.test.ts`).
+
+### Changed
+
+- **`bgutils-js` upgraded 3.2.0 → 4.0.3**, the YouTube PoToken stack's most upstream-brittle dependency and the one pin with no written rationale. v4 is a hard break: the barrel entry is gone (the package now exposes only `/botguard`, `/webpo` and `/utils` subpaths, so the previous bare import resolves to nothing), `BG.Challenge.create` became a free `getChallenge()` whose config drops `globalObj`/`identifier` and renames `fetch` to `fetchFunction`, and `BotGuardClient`'s `globalObj` became `globalObject`. All three renames fail silently rather than loudly — a wrong `globalObject` leaves the VM with no global to attach to and the minter factory returns a non-function — so each is now pinned by a test that asserts the field names v4 actually reads, and the whole flow was re-validated live: a real PoToken minted against Google's attestation endpoint and a real transcript fetched. Staying on v3 meant sitting on an unmaintained line for the one subsystem that breaks whenever YouTube changes BotGuard, with the failure mode being a silent empty transcript body (`src/youtube/potoken.ts`).
+- **`jsdom` deliberately stays at 29.1.1.** jsdom 30 narrows its supported Node range to `^22.22.2 || ^24.15.0 || >=26.0.0`, which excludes both part of this package's own `>=22.19.0` floor and Node 25 entirely; 29's `^20.19.0 || ^22.13.0 || >=24.0.0` covers the whole supported surface. Attempted, measured against the engine ranges, reverted — the pin now has the same recorded rationale as the others (`package.json`).
+
+### Added
+
+- **A full SDK API reference.** `src/sdk.ts` exports 24 symbols; the documentation named 7. `runResearchDetailed`, `scrapeUrl`, `verifyUrl`, `exportKnowledge`, `getSessionMetrics`, `logRunErrorSummary` and the four `getLast*` telemetry accessors were all public and all undiscoverable short of reading the source. Every export now has a signature, a description and — where it differs — a note that it is reachable only through the `/sdk` subpath. A test keeps the two in sync (`docs/SDK.md`, `test/unit/sdk/sdk-docs-surface.test.ts`).
+- **A contract test for the pi-host `ResourceLoader` adapter**, which had 0% coverage and was mocked out in the one test that imported it. This is the seam whose drift under pi 0.81.0 aborted *every* researcher session with "Research produced no report"; the defense against a repeat was a comment. The test runs the real adapter against the installed host: the registration lists the host iterates must be iterable, each resource getter must return the shape the host destructures, and the adapter must implement every public accessor the host's own loader exposes — so a future pi that extends the contract fails here instead of at runtime with a green suite (`test/unit/utils/make-resource-loader.test.ts`).
+
+### Documentation
+
+- README no longer frames read-only operation as a trust caveat. Being unable to run commands or write anything is the correct shape for an agent whose whole job is reading untrusted web pages: prompt injection picked up mid-run has nothing to act on. The cost line also states plainly that LLM tokens are the only cost, and the page reads less in the second person throughout.
+
 ## [1.5.3] - 2026-08-22
 
 ### Fixed

package/docs/SDK.md

@@ -75,15 +75,8 @@ await shutdownResearchSDK();
 
 `initResearchSDK` must run before any research call. Auth resolves from
 `options.apiKey` + `options.provider`, else `process.env.PI_RESEARCH_API_KEY` /
-`PI_RESEARCH_PROVIDER`, else pi's `~/.pi/agent/auth.json`. Other exports include
-`runResearchDetailed`, `searchKnowledge`, `scrapeUrl`, `getResearchHealth`,
-`getLastRunStats`, and `getSessionMetrics`, plus `exportKnowledge` (write the
-knowledge store to a web-consumable JSON file) and the post-run telemetry accessors
-`getLastRunMetrics`, `getLastRunSummary`, `getLastErrorReport`, and
-`getLastResearcherOutcome` (planned/launched/succeeded/failed researcher counts plus
-per-researcher failure reasons for the most recent run — lets a caller tell a thin
-report caused by a sparse topic apart from one where most researchers failed). Both
-`@lincoln504/pi-research` and `@lincoln504/pi-research/sdk` export these symbols.
+`PI_RESEARCH_PROVIDER`, else pi's `~/.pi/agent/auth.json`. The five calls above are the
+common path; [API reference](#api-reference) below lists every export with its signature.
 
 > Concurrency: a single initialized SDK instance runs one research call at a time.
 > Overlapping `runDeepResearch`/`runQuickResearch` calls on the same instance throw
@@ -156,6 +149,76 @@ that run, not the browser pool, LanceDB handles or worker processes.
 The SDK does not write report files. Report export is a front-end concern — the pi
 extension and the CLI / agent skill do it when `PI_RESEARCH_REPORT_EXPORT_ENABLED=true`.
 
+### API reference
+
+Everything below is exported from both `@lincoln504/pi-research` and
+`@lincoln504/pi-research/sdk`, except the two marked *sdk subpath only* — the package
+entry point deliberately does not re-publish them. Every call except `repairJson` and
+`getSDKContainer` requires `initResearchSDK()` first and throws `SDK not initialized`
+otherwise.
+
+**Lifecycle**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `initResearchSDK` | `(options?: ResearchSDKOptions) => Promise<void>` | Registers services. No-ops if already initialized; waits out an in-flight shutdown. |
+| `shutdownResearchSDK` | `() => Promise<void>` | Required. Drains the writer queue, closes LanceDB, kills worker processes. Clears every `getLast*` accessor. |
+| `getSDKContainer` | `() => ServiceContainer \| null` | *sdk subpath only.* Internal/test surface — the live service container, or `null` before init. Not covered by semver; the package entry point does not export it. |
+
+**Research**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `runDeepResearch` | `(query, options?, signal?) => Promise<string>` | Depth 1–3. Returns the Markdown report. `options` is `ResearchOptions` minus the fields the SDK owns (`ctx`, `query`, `model`, `sessionId`, `researchId`). |
+| `runQuickResearch` | `(query, options?, signal?) => Promise<string>` | Depth 0. Same as above with `depth` fixed and therefore not accepted. |
+| `runResearchDetailed` | `(query, options?, signal?) => Promise<ResearchRunResult>` | Same run as `runDeepResearch`, returning `{ report, sessionId, runId, metrics, stats, reports }` instead of the bare string. |
+| `getResearchReports` | `(researchId?) => Promise<Map<string, string>>` | Per-researcher reports keyed by researcher id. Defaults to the most recent run; empty map if there has not been one. |
+
+**Web access**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `scrapeUrl` | `(url, signal?) => Promise<ScrapeResult>` | One URL through the full pipeline (SSRF gate → fetch or stealth browser → PDF extraction → Markdown). |
+| `verifyUrl` | `(url, signal?) => Promise<boolean>` | Reachability only, no content returned. Resolves `false` rather than throwing on a blocked or dead URL. |
+
+**Knowledge store**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `searchKnowledge` | `(queries: string[], signal?) => Promise<KnowledgeSearchResult>` | `{ text, found: 'yes' \| 'maybe' \| 'no', documentsSearched, citations }`. Resolves `found: 'no'` when the store is disabled, empty, or unavailable — it does not throw. |
+| `exportKnowledge` | `(outputPath: string) => Promise<void>` | Writes the store to a web-consumable JSON file. |
+
+**Post-run telemetry** — all reflect the most recent completed run, are `null` until one completes, and are cleared by `shutdownResearchSDK()`.
+
+| Export | Signature | Notes |
+|---|---|---|
+| `getLastRunStats` | `() => ResearchStats \| null` | Headline counts (searches, scrapes, tokens, cost) derived from the run snapshot. |
+| `getLastRunMetrics` | `() => IMetricsSnapshot \| null` | The raw counters/gauges/histograms for that run. |
+| `getLastRunSummary` | `() => RunSummary \| null` | `{ runId, startedAt, completedAt, durationMs, status, snapshot }`. |
+| `getLastErrorReport` | `() => ErrorReport \| null` | Aggregated errors — totals, patterns, by-domain, by-type. Lets an unattended caller see what failed without parsing logs. |
+| `getLastResearcherOutcome` | `() => ResearcherOutcome \| null` | `{ planned, launched, succeeded, failed, failureReasons }`. Distinguishes a thin report caused by a sparse topic from one where most researchers failed. |
+| `getSessionMetrics` | `() => IMetricsSnapshot` | Cumulative across every run since init, rather than per-run. |
+| `logRunErrorSummary` | `(report, depthLabel, status) => void` | *sdk subpath only.* Emits one compact, secret-redacted line for a run's tracked errors. No-op when the report is null or empty. |
+
+**Health and utilities**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `getResearchHealth` | `(opts?: { force?: boolean }) => Promise<HealthReport>` | Runs every registered check. `force` bypasses the cached result. |
+| `repairJson` | `(json: string) => string` | Repairs truncated/malformed model JSON. Pure; usable before init. |
+
+**Types** — `ResearchSDKOptions`, `ResearchRunResult`, `KnowledgeSearchResult`, `ResearcherOutcome`,
+`ResearchOptions`, `ResearchObserver`, `IMetricsSnapshot`, `IMetricHistogram`, `RunSummary`, and
+`ResearchStats`.
+
+**Also on the package entry point**, for callers that want the orchestrators directly rather
+than through the SDK wrapper: `DeepResearchOrchestrator`, `QuickResearchOrchestrator`,
+`HeadlessObserver` (an observer that prints progress to stdout), `ServiceNames`,
+`shutdownManager`, `extractRunStats`, `normalizeUrl`, the config accessors
+`getConfig` / `setConfig` / `resetConfig` / `validateConfig`, and the team-size constants.
+These are lower-level than the SDK functions above and assume you manage the service
+container yourself.
+
 ### Init options
 
 | Option | Description |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.5.3",
+  "version": "1.6.1",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,
@@ -118,7 +118,7 @@
     "@kreuzberg/html-to-markdown-node": "^3.5.5",
     "@lancedb/lancedb": "0.37.1",
     "apache-arrow": "21.1.0",
-    "bgutils-js": "^3.2.0",
+    "bgutils-js": "^4.0.3",
     "camoufox-js": "^0.10.2",
     "esbuild": "^0.28.1",
     "impit": "0.13.0",

package/src/constants.ts

@@ -201,6 +201,36 @@ export const MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
  */
 export const DEFAULT_EXCLUDED_TOOLS: readonly string[] = ['grep'];
 
+/**
+ * Volume signal for {@link isAnalysisFreeSynthesis}: prose below this, measured BEFORE
+ * the CITED LINKS section, is treated as a synthesis containing no analysis.
+ *
+ * Calibrated against 41 real reports from the run logs rather than guessed. The two
+ * broken ones — each a lone title line, delivered as a "complete" 6.8KB and 9.9KB
+ * document because the engine's own citation rebuild padded them out — measured 131 and
+ * 147 characters. The shortest LEGITIMATE report measured 1056.
+ *
+ * Set at 800 rather than hugging the failures, because the response to tripping it is
+ * ADDITIVE: the model's own text is kept and the researcher reports are appended below
+ * it. A false positive therefore costs some duplication, never a lost report, so the
+ * boundary is placed to catch marginal cases rather than to avoid them. It is also only
+ * one of two independent signals — a synthesis with no sentence-ending punctuation is
+ * caught at any length, so a long title would not slip through on volume alone.
+ */
+export const MIN_SYNTHESIS_PROSE_CHARS = 800;
+
+/** Flags a quick-mode report that contains no analysis, where there is no fallback to add. */
+export const QUICK_THIN_REPORT_NOTICE =
+  '_Note: this run did not produce a substantive write-up. The sources below were ' +
+  'retrieved and are cited accurately, but the summary above should not be treated as ' +
+  'a complete answer — consider re-running, or running at a deeper level._';
+
+/** Explains an additive fallback in the delivered report, so the reader is never misled. */
+export const SYNTHESIS_FALLBACK_NOTICE =
+  '_Note: the synthesis step did not produce a usable summary for this run, so the ' +
+  'individual researcher reports are included below in full. The sources cited at the ' +
+  'end of this document are unaffected._';
+
 /**
  * The complete set of excludable research tool names — the vocabulary
  * `--exclude-tools` / `excludeTools` / PI_RESEARCH_DISABLED_TOOLS accept.

package/src/infrastructure/browser/browser-client.ts

@@ -16,6 +16,7 @@ import { errorTracker } from '../../utils/error-tracker.ts';
 import { redactSecrets } from '../../utils/log-utils.ts';
 import { Utf8Body } from '../../utils/http-body.ts';
 import { getClientAgent } from './client-agent.ts';
+import { isTransientSocketError } from './browser-error-utils.ts';
 import { getHealthCheckBudgetMs } from './config.ts';
 import type { NodeError } from '../../types/index.ts';
 
@@ -281,12 +282,29 @@ export class BrowserClient implements IScheduler {
                     errorType = 'unknown';
                 }
                 const error = new Error(errorMsg);
-                logger.error(`[BrowserClient] Request to http://127.0.0.1:${this.port}${path} failed:`, errorMsg);
-                errorTracker.trackError(error, {
-                    component: 'browser-manager',
-                    operation,
-                    errorType,
-                });
+                // This is one ATTEMPT against the leader, not an outcome. runBrowserTask
+                // retries every transient socket failure — a leader handover (ECONNREFUSED
+                // against the departing leader's port) is the common case and recovers a
+                // few hundred milliseconds later. Logging ERROR and tracking here made every
+                // recovered handover look like a fault twice over: once in the ERROR log,
+                // and once more in the error tracker on top of the caller's own richer
+                // record (taskType + errorType) on the retry path. Observed 5 of 5 times in
+                // a real run log, each followed immediately by a successful recovery.
+                //
+                // A failure the caller will NOT retry still reports at ERROR here, and a
+                // retryable one that exhausts its budget still surfaces upstream with URL
+                // context (see web-scraper's isBenignScrapeFailure branch), so nothing that
+                // actually costs the user a result becomes invisible.
+                if (isTransientSocketError(error)) {
+                    logger.debug(`[BrowserClient] Transient failure talking to http://127.0.0.1:${this.port}${path} (caller will retry): ${errorMsg}`);
+                } else {
+                    logger.error(`[BrowserClient] Request to http://127.0.0.1:${this.port}${path} failed:`, errorMsg);
+                    errorTracker.trackError(error, {
+                        component: 'browser-manager',
+                        operation,
+                        errorType,
+                    });
+                }
                 reject(error);
             });
             req.write(JSON.stringify(data));

package/src/infrastructure/browser/browser-error-utils.ts

@@ -124,6 +124,59 @@ export function isPoolShutdownError(error: unknown): boolean {
     );
 }
 
+/** Playwright navigation failures whose cause is at the remote end — see below. */
+const BENIGN_NAVIGATION_FAILURE =
+  /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
+
+/**
+ * True when a scrape failure is an EXPECTED per-URL outcome rather than an engine
+ * fault. Scraping the open web routinely hits bot-protection interstitials, thin
+ * "stub" pages, HTTP 4xx/5xx, slow/unresponsive remotes, and (on quit) pool-drain
+ * races. None of these is a pi-research bug: the URL is already accounted for as
+ * "not scraped" (scrape_results_total{outcome=total_failure}) and surfaced to the
+ * user that way, and no operator action follows. Callers use this to log such
+ * failures at debug instead of error — keeping the ERROR log (and, via
+ * logger.error's error-tracker re-track, the diagnostic error count) focused on
+ * genuine faults. The messages matched here are produced by validateContent
+ * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
+ * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
+ * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
+ * exceeds the deliberate 100MB policy cap, is routine on the open web and
+ * just as non-actionable as a stub page or a 404; the 25MB HTML policy caps
+ * (`HTML response too large` from the fetch layer, `Browser HTML too large`
+ * from the browser layer) are the same deliberate-cap class; the remaining
+ * cases delegate to the shared browser-layer predicates. Deliberately does NOT
+ * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
+ * failure) — that is an infrastructure fault affecting every URL, not a
+ * routine per-PDF outcome, and must still surface at ERROR.
+ *
+ * Navigation failures reported by Playwright (`page.goto: …`) are the same class
+ * once — and only once — the reason is a network, TLS or timeout condition at the
+ * remote end: an expired/self-signed certificate (`SEC_ERROR_UNKNOWN_ISSUER` on a
+ * live university PDF host), a DNS or connection error (`net::ERR_…`,
+ * `NS_ERROR_…`), or a site too slow to answer inside the scrape budget. These were
+ * logged at ERROR by both the leader and the scraper. `page.goto` failures with any
+ * OTHER reason — notably `Target closed`, which is how a crashed or force-closed
+ * browser surfaces — are deliberately excluded and still report as faults.
+ */
+export function isBenignScrapeFailure(error: unknown): boolean {
+  const msg = error instanceof Error ? error.message : String(error);
+  return (
+    msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+    msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
+    msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
+    msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
+    msg.startsWith('Unsupported download:') ||             // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+    BENIGN_NAVIGATION_FAILURE.test(msg) ||                 // remote TLS/DNS/timeout — the URL could not be loaded
+    msg.startsWith('HTML response too large') ||          // fetch-layer 25MB policy cap — same class as the PDF cap
+    msg.startsWith('Browser HTML too large') ||           // browser-layer 25MB policy cap
+    /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
+    isCloudflareBlockError(error) ||
+    isTaskTimeoutError(error) ||
+    isPoolShutdownError(error)
+  );
+}
+
 // Session-scoped circuit breaker storage to support concurrent research runs
 const sessionCircuitBreakers = new Map<string, CircuitBreaker>();
 // Hard cap so a cleanup path that ever skips clearSessionCircuitBreaker (e.g. an earlier

package/src/infrastructure/browser/browser-server.ts

@@ -3,7 +3,7 @@ import * as crypto from 'node:crypto';
 import { logger } from '../../logger.ts';
 import { Utf8Body } from '../../utils/http-body.ts';
 import type { SearchResult } from '../../web-research/types.ts';
-import { isCloudflareBlockError, isPoolShutdownError } from './browser-error-utils.ts';
+import { isCloudflareBlockError, isPoolShutdownError, isBenignScrapeFailure } from './browser-error-utils.ts';
 
 export interface BrowserServerOptions {
     onSearch: (query: string, signal?: AbortSignal) => Promise<SearchResult[]>;
@@ -204,6 +204,14 @@ export class BrowserServer {
                             // Pool destroyed mid-request during quit/SIGTERM teardown — expected,
                             // not a server fault. DEBUG so a normal shutdown doesn't emit ERRORs.
                             logger.debug('[BrowserServer] Request abandoned during shutdown:', error instanceof Error ? error.message : String(error));
+                        } else if (isBenignScrapeFailure(error)) {
+                            // Per-URL outcomes the CLIENT side already classifies as routine —
+                            // an HTTP 4xx/5xx, a navigation timeout, an oversized PDF, a blocked
+                            // fetch. The leader saw the same failure first and logged it at ERROR
+                            // regardless, so every one of those outcomes still produced exactly
+                            // one false ERROR per URL despite the client's classification. Same
+                            // predicate as web-scraper.ts, so the two layers agree.
+                            logger.debug('[BrowserServer] Expected per-URL scrape outcome:', error instanceof Error ? error.message.split('\n')[0] : String(error));
                         } else {
                             logger.error('[BrowserServer] Error handling request:', error);
                         }

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -368,7 +368,10 @@ export async function initBrowser(): Promise<void> {
  * rather than left to leak.
  */
 export async function acquireTaskContext(): Promise<any> {
-  const contextPromise = browser.newContext();
+  // acceptDownloads is explicit, not defaulted: executeScrapeTask recovers PDFs that a
+  // server sends as an attachment by reading the resulting Download, and that object
+  // only exists when downloads are accepted.
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId: NodeJS.Timeout | undefined;
   let contextRaceLost = false;