@lincoln504/pi-research 1.5.2 → 1.6.0

package/dist/thread-worker.mjs

@@ -1516,11 +1516,25 @@ function isPoolShutdownError(error) {
   // same transient swap the "being reset" string covers, not a hard fault.
   err.message.includes("Cannot execute a task on not started pool"));
 }
-var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
+function isBenignScrapeFailure(error) {
+  const msg = error instanceof Error ? error.message : String(error);
+  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
+  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
+  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
+  msg.startsWith("Unsupported download:") || // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+  BENIGN_NAVIGATION_FAILURE.test(msg) || // remote TLS/DNS/timeout — the URL could not be loaded
+  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
+  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
+  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
+  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
+}
+var BENIGN_NAVIGATION_FAILURE, DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({
   "src/infrastructure/browser/browser-error-utils.ts"() {
     "use strict";
     init_circuit_breaker();
+    BENIGN_NAVIGATION_FAILURE = /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
     DEFAULT_BREAKER_CONFIG = {
       // Raised from 3→8: parallel search bursts of 20 queries frequently produce
       // simultaneous timeouts on slow/CF-protected sites. A threshold of 3 caused
@@ -1565,17 +1579,6 @@ import * as dnsCb from "node:dns";
 import * as net from "node:net";
 import crypto from "node:crypto";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-function isBenignScrapeFailure(error) {
-  const msg = error instanceof Error ? error.message : String(error);
-  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
-  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
-  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
-  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
-  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
-  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
-  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
-}
 function getRandomUserAgent() {
   try {
     const index = crypto.randomInt(0, USER_AGENTS.length);
@@ -2014,7 +2017,7 @@ import { platform as platform2 } from "node:os";
 
 // src/infrastructure/browser/thread-worker-messaging.ts
 init_log_utils();
-import { appendFileSync as appendFileSync2 } from "node:fs";
+import { appendFileSync as appendFileSync2, promises as fsp } from "node:fs";
 
 // src/utils/safe-unref.ts
 function safeUnref(timer) {
@@ -2286,6 +2289,66 @@ function seedSubresourceVerdict(validatedUrl) {
   } catch {
   }
 }
+function isEvictedBodyError(err) {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+var PDF_REFETCH_TIMEOUT_MS = 9e4;
+async function refetchPdfBody(page, url, validateForWorker, signal) {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error("Aborted");
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS
+  });
+  requestPromise.catch(() => {
+  });
+  const response = await (signal ? Promise.race([
+    requestPromise,
+    new Promise((_, reject) => {
+      if (signal.aborted) return reject(new Error("Aborted"));
+      signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
+    })
+  ]) : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h) => String(h.name).toLowerCase() === "content-length")?.value ?? ""), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+  const buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+async function readPdfDownload(download, url, page, pendingAddrChecks, getPoisonedError) {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+  const suggested = String(download.suggestedFilename?.() ?? "");
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf = suggested.toLowerCase().endsWith(".pdf") || downloadUrl.toLowerCase().split("?")[0].endsWith(".pdf");
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+  const filePath = await download.path();
+  if (!filePath) throw new Error("Fetch blocked: download produced no readable file");
+  const { size } = await fsp.stat(filePath);
+  if (size > MAX_PDF_SIZE) {
+    const sizeMB = Math.round(size / 1024 / 1024);
+    throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+  }
+  const buffer = await fsp.readFile(filePath);
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+  await download.delete?.().catch(() => {
+  });
+  await page.close();
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: "application/pdf", bufferB64: buffer.toString("base64"), jitter: 0 };
+}
 async function executeScrapeTask(_context, url, signal) {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
@@ -2302,6 +2365,7 @@ async function executeScrapeTask(_context, url, signal) {
   let poisonedError = null;
   const pendingAddrChecks = [];
   let oversizedPdfError = null;
+  let pendingDownload = null;
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
     const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2368,7 +2432,25 @@ async function executeScrapeTask(_context, url, signal) {
       } catch {
       }
     });
-    const response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    page.on("download", (d) => {
+      pendingDownload ??= d;
+    });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    } catch (navErr) {
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent("download", { timeout: 5e3 }).catch(() => null),
+          new Promise((r) => setTimeout(r, 5e3))
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
     const serverAddr = await response?.serverAddr().catch(() => null);
     if (serverAddr?.ipAddress) {
       const ip = serverAddr.ipAddress;
@@ -2386,7 +2468,13 @@ async function executeScrapeTask(_context, url, signal) {
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       if (buffer.byteLength > MAX_PDF_SIZE) {
         const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
@@ -2749,7 +2837,7 @@ function logToDebugFile3(level, ...args) {
         return String(arg);
       }).join(" "))
     };
-    import("node:fs/promises").then((fsp) => fsp.appendFile(logFile, `${JSON.stringify(entry)}
+    import("node:fs/promises").then((fsp2) => fsp2.appendFile(logFile, `${JSON.stringify(entry)}
 `)).catch(() => {
     });
   } catch {
@@ -2888,7 +2976,7 @@ async function initBrowser() {
   return initPromise;
 }
 async function acquireTaskContext() {
-  const contextPromise = browser.newContext();
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId;
   let contextRaceLost = false;

package/docs/AGENT-SKILL.md

@@ -1,9 +1,9 @@
 ## Agent Skill
 
 pi-research ships as a portable [Agent Skill](https://agentskills.io/specification)
-so any skills-compatible coding agent (Claude, OpenAI Codex CLI, OpenClaw, and
-others using the same `SKILL.md` directory model) can run web research through the
-pi-research software.
+so any skills-compatible coding agent using the same `SKILL.md` directory model —
+Claude, OpenAI Codex CLI and others — can run web research through the pi-research
+software.
 
 ### Install
 
@@ -18,7 +18,7 @@ every coding agent detected on this machine:
 
 ```bash
 npm install -g @lincoln504/pi-research   # the engine (puts `pi-research` on PATH)
-pi-research skill install                # link the skill into Claude, Codex, OpenClaw
+pi-research skill install                # link the skill into every detected agent
 ```
 
 `skill install` targets only agents already set up under `$HOME`, never overwrites a
@@ -49,7 +49,7 @@ disabled"); or run `Set-ExecutionPolicy -Scope CurrentUser RemoteSigned` once.
 agent
   │  shells out (Bash / exec)
   ▼
-run.mjs  —  zero-dep launcher (skills/pi-research/scripts/)
+run.mjs  —  zero-dep launcher (agent-skill/pi-research/scripts/)
   │  locates the installed engine, or fails fast with guidance
   ▼
 pi-research engine  —  the CLI (dist/cli.mjs)
@@ -70,19 +70,23 @@ is missing. It exposes four subcommands: `research "<query>"` (live research),
 
 ### Installation flow
 
-The skill source lives at `skills/pi-research/` inside the package. Installing means
-linking that directory into each agent's skills folder.
+The skill source lives at `agent-skill/pi-research/` inside the package. Installing
+means linking that directory into each agent's skills folder.
+
+> The directory is deliberately **not** named `skills/`: `pi` treats a package-root
+> `skills/` directory as one of its own resource roots and will load what it finds
+> there, which would shadow the extension's native research tool with a slower
+> subprocess copy of itself.
 
 One-click (recommended). From the `pi` extension, run `/research-config` →
 Install in External Agents. The installer:
 
 1. Detects which target agents are present under `$HOME` — currently Claude
-   (`~/.claude/skills`), OpenClaw (`~/.openclaw/skills`, OpenClaw's managed
-   skill root, which accepts symlinked skill folders), and OpenAI Codex CLI
-   (`~/.codex/skills` — unlike the other two, this path is not confirmed by
-   Codex's official docs; Codex skills support is still emerging).
-2. Symlinks `skills/pi-research/` into each present agent, never overwriting an
-   unrelated skill already in that slot.
+   (`~/.claude/skills`), OpenAI Codex CLI (`~/.codex/skills` — this path is not
+   confirmed by Codex's official docs; Codex skills support is still emerging)
+   and OpenClaw (`~/.openclaw/skills`).
+2. Symlinks `agent-skill/pi-research/` into each present agent, never overwriting
+   an unrelated skill already in that slot.
 3. Records what it created in a manifest, so Remove from External Agents
    removes only its own links. Stale links are also garbage-collected on startup.
 
@@ -99,29 +103,12 @@ do exactly the same from the CLI — same agent detection, same manifest, same
 never-clobber-a-foreign-skill guarantee — for people who installed the engine with
 `npm install -g` and never open the interactive extension.
 
-**OpenClaw without `pi`**
-
-If you run OpenClaw but not the `pi` extension, register the skill with OpenClaw's
-own CLI after installing the engine — OpenClaw copies the `SKILL.md` folder into its
-managed skill root and reads it on the next session:
-
-```bash
-npm install -g @lincoln504/pi-research
-openclaw skills install "$(npm root -g)/@lincoln504/pi-research/skills/pi-research" --global
-```
-
-OpenClaw drives the skill through its `exec` tool, gated on `node` being on `PATH`
-(`metadata.openclaw.requires.bins`). A bare `openclaw skills install git:…` is *not*
-used here: OpenClaw expects `SKILL.md` at a repo root, whereas this package nests it
-under `skills/pi-research/` — install the engine from npm and point OpenClaw at that
-folder, as above.
-
-Cursor is intentionally excluded from the one-click flow — it has no global skills
-directory and reads only project-level `.cursor/skills/`. Link it per project:
-
-```bash
-ln -s "$(npm root -g)/@lincoln504/pi-research/skills/pi-research" .cursor/skills/pi-research
-```
+An agent with its own skill-registration CLI can be pointed at the shipped folder
+instead of symlinking it. Install the engine, then register
+`$(npm root -g)/@lincoln504/pi-research/agent-skill/pi-research` with that agent —
+it holds `SKILL.md` at its root, which is the layout such tools expect. Agents that
+copy rather than link pick up engine upgrades on the next `skill install`, not
+automatically.
 
 Manual. Symlink the directory into any agent's skills folder yourself:
 
@@ -130,7 +117,6 @@ Manual. Symlink the directory into any agent's skills folder yourself:
 | Claude | `~/.claude/skills/pi-research/` | `<project>/.claude/skills/pi-research/` |
 | OpenAI Codex CLI | `~/.codex/skills/pi-research/` | `<project>/.codex/skills/pi-research/` |
 | OpenClaw | `~/.openclaw/skills/pi-research/` | `<workspace>/skills/pi-research/` |
-| Cursor | — (no global dir) | `<project>/.cursor/skills/pi-research/` |
 
 ### Prerequisites
 
@@ -146,6 +132,6 @@ node "<skill_dir>/scripts/run.mjs" status   # verify the engine is detected
 ![One-command health and readiness check](https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/06-health-check.gif)
 
 Once installed, ask the agent to research something — its skill system activates
-pi-research automatically. The in-package readme (`skills/pi-research/README.md`) and
-`skills/pi-research/references/configuration.md` carry the same detail for anyone
-browsing the skill directly.
+pi-research automatically. The in-package readme (`agent-skill/pi-research/README.md`)
+and `agent-skill/pi-research/references/configuration.md` carry the same detail for
+anyone browsing the skill directly.

package/docs/ARCHITECTURE.md

@@ -4,7 +4,7 @@ pi-research is a pi TUI extension for multi-agent web research. It runs inside t
 process, registers its tools and commands, and manages its own browser worker pool,
 service registry, and local knowledge store. One engine backs every front-end: besides
 the pi extension it is exposed as a standalone CLI, a portable agent skill (the same
-skill any skills-aware host runs, including OpenClaw) and a programmatic SDK
+skill any skills-aware host runs) and a programmatic SDK
 (`src/sdk.ts`).
 
 ```

package/docs/CHANGELOG.md

@@ -5,6 +5,58 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.0] - 2026-08-22
+
+Driven by log forensics rather than by the test suite: every entry below was found
+by reading what the engine actually reported in production, and each fix was
+re-measured against the exact URL, error line or counter that exposed it.
+
+The starting observation was that the ERROR log had stopped meaning anything. Across
+the retained run logs, **all 26 ERROR lines were either a condition the engine had
+already recovered from or a routine per-URL outcome** — a blocked page, a slow site,
+an untrusted certificate. Not one described an unrecovered engine fault. Two classes
+account for all of them: content the scraper had already fetched and then dropped,
+and recovered conditions reported as terminal failures.
+
+### Fixed
+
+- **PDF sources the browser had already fetched were being discarded two different ways, both now recovered.** A server sending the PDF with `Content-Disposition: attachment` makes Firefox route the response to the download manager, so no document is produced and `page.goto()` rejects with "Download is starting" — the PDF branch was never reached even though the bytes had arrived. Separately, Firefox discards large navigation response bodies from its juggler-side cache, after which `response.body()` fails with "Response body … was evicted!". Both recur across the run logs, and both hit exactly the sources research most wants — vendor datasheets, standards-body documents, journal and university PDFs. The download is now read back from disk, and an evicted body is re-requested through the browser context's own request API — which carries the same cookies, headers, proxy and TLS profile as the navigation, the reason the URL reached the stealth-browser fallback at all. Verified live against the two URLs that failed: a Frontiers article PDF now yields 106k characters of text and a 13.9MB TU Delft lecture PDF 27k, both previously total failures. The re-fetch was measured before it was written: asking for the body earlier does not help, because the eviction happens when the browser drops the bytes, not when we ask for them — a first attempt built on that assumption was discarded after a direct camoufox probe disproved it (`src/infrastructure/browser/thread-worker-messaging.ts`, `src/infrastructure/browser/thread-worker-browser.ts`).
+- **A knowledge-store write that was retried and succeeded no longer reports as a failed write.** `addDocumentsToStore` logged ERROR and incremented `knowledge_store_add_documents_total{status=error}` before rethrowing — but its caller catches exactly that throw, reconnects to a fresh embedder and retries, so the write usually completed. All 8 occurrences across the retained logs are immediately followed by the reconnect WARN — 8 of 8 recovered, none lost — and every one of them counted the same batch twice (once error, once success). The Lance commit-conflict retry fed the same path, so cross-process write contention did it too. Attempt-level accounting now stays in the attempt and outcome accounting moved to `KnowledgeStore.addDocuments`, which is the only layer that knows whether a write was actually abandoned — exactly one counter increment per call, and an ERROR only when the batch is genuinely lost (`src/knowledge/store-operations.ts`, `src/knowledge/store.ts`).
+- **The browser leader logged every routine per-URL scrape outcome as a server fault.** `BrowserServer`'s handler classified Cloudflare challenges and pool-drain races but sent everything else to ERROR, including the HTTP 4xx/5xx, navigation timeouts and over-cap PDFs that the client side already classifies as expected. The leader saw each failure first, so the client's careful classification never prevented the ERROR — one false fault per affected URL. Both layers now share the same predicate (`src/infrastructure/browser/browser-server.ts`).
+- **A leader handover was reported as a terminal error by the layer that was about to retry it.** `BrowserClient` logged ERROR and recorded a tracked error for any socket failure, including the ECONNREFUSED a follower gets against a leader that has already exited. `runBrowserTask` retries that within a second: 5 of 5 such lines in the log were followed immediately by the recovery WARN, and each was counted twice in the diagnostic error report — once here without context, once by the caller with it. A failure the caller will not retry still reports at ERROR, and a retryable one that exhausts its budget still surfaces upstream with URL context (`src/infrastructure/browser/browser-client.ts`).
+- **Remote TLS, DNS and navigation-timeout failures are now classified as the per-URL outcomes they are.** `page.goto: SEC_ERROR_UNKNOWN_ISSUER` (a live university PDF host with an untrusted certificate) and `page.goto: Timeout 15000ms exceeded` reached the ERROR log and the diagnostic error count from both the leader and the scraper, though neither is actionable and the URL is already tallied as not scraped. `page.goto: Target closed` — how a crashed or force-closed browser surfaces — is deliberately excluded and still reports as a fault (`src/web-research/scraper-utils.ts`).
+
+- **The unit suite was writing its deliberate failure fixtures into the user’s real shared log.** `unit-env.ts` redirects `PI_RESEARCH_LOG_PATH` before `src/logger.ts` is evaluated, but the global logger is built lazily and resolves its path at CONSTRUCTION — so a suite whose `beforeEach` deletes every `PI_RESEARCH_*` variable (config.test.ts did) left the first logger of the run binding to the production default instead. Measured at ~7KB per run appended to `/tmp/pi-research.log`, including a `{corrupt json` registry fixture whose ERROR line reads exactly like a live failure and cost real time during this release’s own forensic pass. The setup now constructs the logger while the redirect is still set, config.test.ts keeps the redirect out of its scrub, and a test pins the guarantee; whole-suite pollution measured at 0 bytes after the fix (`test/setup/unit.ts`, `test/unit/config.test.ts`, `test/unit/logger-test-isolation.test.ts`).
+
+### Changed
+
+- **`bgutils-js` upgraded 3.2.0 → 4.0.3**, the YouTube PoToken stack's most upstream-brittle dependency and the one pin with no written rationale. v4 is a hard break: the barrel entry is gone (the package now exposes only `/botguard`, `/webpo` and `/utils` subpaths, so the previous bare import resolves to nothing), `BG.Challenge.create` became a free `getChallenge()` whose config drops `globalObj`/`identifier` and renames `fetch` to `fetchFunction`, and `BotGuardClient`'s `globalObj` became `globalObject`. All three renames fail silently rather than loudly — a wrong `globalObject` leaves the VM with no global to attach to and the minter factory returns a non-function — so each is now pinned by a test that asserts the field names v4 actually reads, and the whole flow was re-validated live: a real PoToken minted against Google's attestation endpoint and a real transcript fetched. Staying on v3 meant sitting on an unmaintained line for the one subsystem that breaks whenever YouTube changes BotGuard, with the failure mode being a silent empty transcript body (`src/youtube/potoken.ts`).
+- **`jsdom` deliberately stays at 29.1.1.** jsdom 30 narrows its supported Node range to `^22.22.2 || ^24.15.0 || >=26.0.0`, which excludes both part of this package's own `>=22.19.0` floor and Node 25 entirely; 29's `^20.19.0 || ^22.13.0 || >=24.0.0` covers the whole supported surface. Attempted, measured against the engine ranges, reverted — the pin now has the same recorded rationale as the others (`package.json`).
+- **Tags now produce GitHub Releases.** 44 tags existed and zero Releases, so each version's changelog entry was reachable only by reading `docs/CHANGELOG.md` in the tree. A new job extracts that version's section from the tag's own copy of the changelog and publishes it. It runs after `publish` and needs it, so a failed npm publish cannot announce a version nobody can install; it holds `contents: write` and nothing else — in particular no `id-token` — and installs nothing, so no dependency lifecycle script runs while that token is live. A version that reached npm with no changelog section fails the job rather than publishing an empty Release (`.github/workflows/release.yml`).
+
+### Added
+
+- **A full SDK API reference.** `src/sdk.ts` exports 24 symbols; the documentation named 7. `runResearchDetailed`, `scrapeUrl`, `verifyUrl`, `exportKnowledge`, `getSessionMetrics`, `logRunErrorSummary` and the four `getLast*` telemetry accessors were all public and all undiscoverable short of reading the source. Every export now has a signature, a description and — where it differs — a note that it is reachable only through the `/sdk` subpath. A test keeps the two in sync (`docs/SDK.md`, `test/unit/sdk/sdk-docs-surface.test.ts`).
+- **A contract test for the pi-host `ResourceLoader` adapter**, which had 0% coverage and was mocked out in the one test that imported it. This is the seam whose drift under pi 0.81.0 aborted *every* researcher session with "Research produced no report"; the defense against a repeat was a comment. The test runs the real adapter against the installed host: the registration lists the host iterates must be iterable, each resource getter must return the shape the host destructures, and the adapter must implement every public accessor the host's own loader exposes — so a future pi that extends the contract fails here instead of at runtime with a green suite (`test/unit/utils/make-resource-loader.test.ts`).
+
+### Documentation
+
+- README no longer frames read-only operation as a trust caveat. Being unable to run commands or write anything is the correct shape for an agent whose whole job is reading untrusted web pages: prompt injection picked up mid-run has nothing to act on. The cost line also states plainly that LLM tokens are the only cost, and the page reads less in the second person throughout.
+
+## [1.5.3] - 2026-08-22
+
+### Fixed
+
+- **The shipped agent skill moved from `skills/` to `agent-skill/` inside the package, so `pi` can no longer load it as one of its own agent skills.** `pi` reserves four package-root resource directory names (`extensions`, `skills`, `prompts`, `themes`) and convention-scans any of them a package's `pi` manifest does not explicitly declare. This package declares `extensions` only, so its root-level `skills/` was scanned and `SKILL.md` registered as a pi agent skill — a slower subprocess duplicate of the extension's own native `research` tool, plus a redundant `/skill:pi-research` command. Nothing fired on a plain install, a plain update, or `npm install`ing the package into a project (pi never scans project `node_modules`, and a settings entry written as a plain string takes the manifest-only path). The trap was `pi config`: toggling **any** pi-research resource rewrites the settings entry from a string into an object, after which pi resolves each resource type separately and falls back to the convention scan for the undeclared `skills` — permanently, since the TUI's revert-to-string branch is unreachable, and irrespective of whether the toggle *enabled* or *disabled* the extension. Renaming the directory removes the scanned path entirely, which needs no `pi.skills` manifest key and so cannot mislabel the package as a skill provider in the pi.dev gallery (`agent-skill/`, `package.json`).
+- **Ownership detection now recognises both the current and the pre-1.5.3 skill source directories, so existing installs survive the move.** Installed links are symlinks into the package, and three of the four ownership call sites pass no expected source — they match the link destination's shape instead. Accepting only the new name would have stranded every link created before this release (reconcile refusing to re-point it, uninstall refusing to remove it, `skill status` reporting a foreign skill); accepting only the legacy name would have made each link unmanageable the moment it was successfully re-pointed, a failure that looks like a clean migration until the *next* upgrade. `reconcileSkillInstalls` runs on every CLI invocation and every extension activation, so a pre-1.5.3 link is re-pointed automatically with no user action. Both failure directions are pinned by a new regression suite, and the false-positive guard for lookalike paths (`…/pi-researcher/…`) is retained (`src/skill-install/skill-installer.ts`, `scripts/cleanup.cjs`, `test/unit/skill-install/legacy-source-dir-migration.test.ts`).
+- The installer's `pi` harness entry pointed at `~/.pi/skills`, but pi's skill root is `~/.pi/agent/skills` — the entry had never matched anything. Corrected to the real path. It remains detection-only and is deliberately not an install target: pi already exposes research natively, so a skill there would only add a slower duplicate (`src/skill-install/skill-installer.ts`).
+
+### Changed
+
+- **Cursor is no longer documented or defined as a harness.** It was never an install target — it exposes no personal/global skills directory, so the one-click flow could not serve it — and its only presence was manual per-project symlink instructions plus an unused entry in the harness table. Nothing installed to it, so there is no behavioural change (`src/skill-install/skill-installer.ts`, `docs/AGENT-SKILL.md`, `agent-skill/pi-research/README.md`).
+- **OpenClaw is de-emphasised throughout the documentation**, becoming one supported agent among others rather than a separately-branded integration: its dedicated install section, per-harness prose and README callouts are gone, and it now appears only where the supported-agent set is enumerated factually. Support is unchanged — it remains a first-class install target with its `SKILL.md` preflight metadata intact, and existing installs are untouched. (Removing it from the target list was considered and rejected: uninstall and status are both target-filtered, so a formerly-supported harness would have become permanently unremovable through any user-facing command while reconcile kept maintaining the orphan.) The generic copy-install route it used is still documented, without naming a specific agent (`README.md`, `docs/`, `agent-skill/pi-research/`).
+- **npm `keywords` trimmed from 25 to 13**, dropping the entries that blurred what pi actually loads from this package (`skills`, `agent-skills`, plus generic SEO filler: `ai-research`, `research`, `scraping`, `browser-automation`, `stealth-browser`, `camoufox`, `duckduckgo`, `llm-tools`, `web-access`, `agent`). Retained: the pi-ecosystem flags (`pi`, `pi-package` — load-bearing for pi.dev gallery inclusion — `pi-extension`, `pi-coding-agent`), the harnesses the standalone skill genuinely targets (`agent-skill`, `claude-code`, `codex`, `openclaw`), and the core function words (`web-research`, `deep-research`, `web-search`, `web-scraping`, `knowledge-store`). Cosmetic only: pi never reads npm keywords — resource loading is driven entirely by the `pi` manifest keys, and the pi.dev type chips derive from those same keys (`package.json`).
+
 ## [1.5.2] - 2026-08-22
 
 ### Fixed

package/docs/CONFIGURATION.md

@@ -1,6 +1,6 @@
 ## Configuration
 
-Every front-end (the pi extension, the standalone CLI / agent skill that OpenClaw
+Every front-end (the pi extension, the standalone CLI / agent skill that Claude Code
 and other skills-aware hosts run, and the SDK) shares one configuration
 model. This document covers the
 settings exposed in the `/research-config` TUI first, then the complete
@@ -159,7 +159,7 @@ Diagnostics & platform
 | `CAMOUFOX_INSTALL_DIR` | _(user cache)_ | camoufox-js's own variable, and the only one that can relocate the browser. Sets where pi-research looks the binary up, and is exported to the postinstall fetch and to browser workers. **Not effective on the currently pinned camoufox-js (0.10.x), which hardcodes its cache directory and honours no variable** — so relocating the browser is unsupported until that pin can advance (see ARCHITECTURE.md). |
 | `PLAYWRIGHT_BROWSERS_PATH` | _(user cache)_ | Alias for the above, accepted for compatibility and exported onward as `CAMOUFOX_INSTALL_DIR`. Same caveat: on the pinned camoufox-js it moves only where pi-research looks, not where the download lands, so setting it reports the browser missing from the custom path after a successful install. |
 | `XDG_CACHE_HOME` | `~/.cache` | Standard XDG variable. When set, every `~/.cache/pi-research/...` path below is rooted at `$XDG_CACHE_HOME/pi-research/...` instead. |
-| `PI_RESEARCH_BIN` (alias `PI_RESEARCH_PATH`) | _(auto)_ | Agent-skill launcher only: explicit path to the pi-research engine binary when auto-resolution (PATH → local install → npx) should be bypassed. See [skills/pi-research/references/configuration.md](../skills/pi-research/references/configuration.md). |
+| `PI_RESEARCH_BIN` (alias `PI_RESEARCH_PATH`) | _(auto)_ | Agent-skill launcher only: explicit path to the pi-research engine binary when auto-resolution (PATH → local install → npx) should be bypassed. See [agent-skill/pi-research/references/configuration.md](../agent-skill/pi-research/references/configuration.md). |
 | `PLAYWRIGHT_INSTALL_DEPS` | _(unset)_ | Linux only. Set `true` during `npm install` to also install system libraries via `npx playwright install-deps` (same as `npm run install:system-deps`). |
 | `PI_RESEARCH_STRICT_SETUP` | _(unset)_ | Read by the bundled `scripts/setup.cjs` during `npm install`. Set `1`/`true` to make a failed browser download fail the install instead of deferring it to first use. |
 | `PI_RESEARCH_CONFIG_DIR_NAME` | `.pi` | Override the host config-directory name under your home dir (advanced; e.g. set to share another harness's config root). |
@@ -196,8 +196,8 @@ layered over the shared base, so they can be configured independently. Exactly t
 exist:
 
 - `~/.pi/research/pi.env` — the pi extension
-- `~/.pi/research/cli.env` — the standalone CLI / agent skill (the surface OpenClaw
-  and other skills-aware hosts run)
+- `~/.pi/research/cli.env` — the standalone CLI / agent skill (the surface
+  skills-aware hosts run)
 
 The overlay files do not exist by default; create the one you need by hand. There is
 intentionally no `sdk.env`: the SDK is a library configured from code (see

package/docs/PI-EXTENSION.md

@@ -78,7 +78,7 @@ environment-variable list live in [CONFIGURATION.md](CONFIGURATION.md).
 ### Coding-agent skill installer
 
 The `/research-config` menu can install the `pi-research` skill into your other
-coding agents (Claude, Codex, OpenClaw) so they can run web research through the
+coding agents detected on this machine so they can run web research through the
 CLI, and remove it again — with exact, manifest-tracked cleanup. See
 [AGENT-SKILL.md](AGENT-SKILL.md) for the full installation flow.
 

package/docs/SDK.md

@@ -75,15 +75,8 @@ await shutdownResearchSDK();
 
 `initResearchSDK` must run before any research call. Auth resolves from
 `options.apiKey` + `options.provider`, else `process.env.PI_RESEARCH_API_KEY` /
-`PI_RESEARCH_PROVIDER`, else pi's `~/.pi/agent/auth.json`. Other exports include
-`runResearchDetailed`, `searchKnowledge`, `scrapeUrl`, `getResearchHealth`,
-`getLastRunStats`, and `getSessionMetrics`, plus `exportKnowledge` (write the
-knowledge store to a web-consumable JSON file) and the post-run telemetry accessors
-`getLastRunMetrics`, `getLastRunSummary`, `getLastErrorReport`, and
-`getLastResearcherOutcome` (planned/launched/succeeded/failed researcher counts plus
-per-researcher failure reasons for the most recent run — lets a caller tell a thin
-report caused by a sparse topic apart from one where most researchers failed). Both
-`@lincoln504/pi-research` and `@lincoln504/pi-research/sdk` export these symbols.
+`PI_RESEARCH_PROVIDER`, else pi's `~/.pi/agent/auth.json`. The five calls above are the
+common path; [API reference](#api-reference) below lists every export with its signature.
 
 > Concurrency: a single initialized SDK instance runs one research call at a time.
 > Overlapping `runDeepResearch`/`runQuickResearch` calls on the same instance throw
@@ -141,7 +134,7 @@ exit code is the part you can rely on.
 
 Treat any code ≥ 128 as a cancellation; `pi-research --help` lists the full set, and
 the agent-facing contract is the exit-code table in
-[`SKILL.md`](../skills/pi-research/SKILL.md).
+[`SKILL.md`](../agent-skill/pi-research/SKILL.md).
 
 These are deliberately not the `70` runtime-error code and never carry
 `retryable: true` — a cancel is a completed intention, not a fault to retry. The
@@ -156,6 +149,76 @@ that run, not the browser pool, LanceDB handles or worker processes.
 The SDK does not write report files. Report export is a front-end concern — the pi
 extension and the CLI / agent skill do it when `PI_RESEARCH_REPORT_EXPORT_ENABLED=true`.
 
+### API reference
+
+Everything below is exported from both `@lincoln504/pi-research` and
+`@lincoln504/pi-research/sdk`, except the two marked *sdk subpath only* — the package
+entry point deliberately does not re-publish them. Every call except `repairJson` and
+`getSDKContainer` requires `initResearchSDK()` first and throws `SDK not initialized`
+otherwise.
+
+**Lifecycle**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `initResearchSDK` | `(options?: ResearchSDKOptions) => Promise<void>` | Registers services. No-ops if already initialized; waits out an in-flight shutdown. |
+| `shutdownResearchSDK` | `() => Promise<void>` | Required. Drains the writer queue, closes LanceDB, kills worker processes. Clears every `getLast*` accessor. |
+| `getSDKContainer` | `() => ServiceContainer \| null` | *sdk subpath only.* Internal/test surface — the live service container, or `null` before init. Not covered by semver; the package entry point does not export it. |
+
+**Research**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `runDeepResearch` | `(query, options?, signal?) => Promise<string>` | Depth 1–3. Returns the Markdown report. `options` is `ResearchOptions` minus the fields the SDK owns (`ctx`, `query`, `model`, `sessionId`, `researchId`). |
+| `runQuickResearch` | `(query, options?, signal?) => Promise<string>` | Depth 0. Same as above with `depth` fixed and therefore not accepted. |
+| `runResearchDetailed` | `(query, options?, signal?) => Promise<ResearchRunResult>` | Same run as `runDeepResearch`, returning `{ report, sessionId, runId, metrics, stats, reports }` instead of the bare string. |
+| `getResearchReports` | `(researchId?) => Promise<Map<string, string>>` | Per-researcher reports keyed by researcher id. Defaults to the most recent run; empty map if there has not been one. |
+
+**Web access**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `scrapeUrl` | `(url, signal?) => Promise<ScrapeResult>` | One URL through the full pipeline (SSRF gate → fetch or stealth browser → PDF extraction → Markdown). |
+| `verifyUrl` | `(url, signal?) => Promise<boolean>` | Reachability only, no content returned. Resolves `false` rather than throwing on a blocked or dead URL. |
+
+**Knowledge store**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `searchKnowledge` | `(queries: string[], signal?) => Promise<KnowledgeSearchResult>` | `{ text, found: 'yes' \| 'maybe' \| 'no', documentsSearched, citations }`. Resolves `found: 'no'` when the store is disabled, empty, or unavailable — it does not throw. |
+| `exportKnowledge` | `(outputPath: string) => Promise<void>` | Writes the store to a web-consumable JSON file. |
+
+**Post-run telemetry** — all reflect the most recent completed run, are `null` until one completes, and are cleared by `shutdownResearchSDK()`.
+
+| Export | Signature | Notes |
+|---|---|---|
+| `getLastRunStats` | `() => ResearchStats \| null` | Headline counts (searches, scrapes, tokens, cost) derived from the run snapshot. |
+| `getLastRunMetrics` | `() => IMetricsSnapshot \| null` | The raw counters/gauges/histograms for that run. |
+| `getLastRunSummary` | `() => RunSummary \| null` | `{ runId, startedAt, completedAt, durationMs, status, snapshot }`. |
+| `getLastErrorReport` | `() => ErrorReport \| null` | Aggregated errors — totals, patterns, by-domain, by-type. Lets an unattended caller see what failed without parsing logs. |
+| `getLastResearcherOutcome` | `() => ResearcherOutcome \| null` | `{ planned, launched, succeeded, failed, failureReasons }`. Distinguishes a thin report caused by a sparse topic from one where most researchers failed. |
+| `getSessionMetrics` | `() => IMetricsSnapshot` | Cumulative across every run since init, rather than per-run. |
+| `logRunErrorSummary` | `(report, depthLabel, status) => void` | *sdk subpath only.* Emits one compact, secret-redacted line for a run's tracked errors. No-op when the report is null or empty. |
+
+**Health and utilities**
+
+| Export | Signature | Notes |
+|---|---|---|
+| `getResearchHealth` | `(opts?: { force?: boolean }) => Promise<HealthReport>` | Runs every registered check. `force` bypasses the cached result. |
+| `repairJson` | `(json: string) => string` | Repairs truncated/malformed model JSON. Pure; usable before init. |
+
+**Types** — `ResearchSDKOptions`, `ResearchRunResult`, `KnowledgeSearchResult`, `ResearcherOutcome`,
+`ResearchOptions`, `ResearchObserver`, `IMetricsSnapshot`, `IMetricHistogram`, `RunSummary`, and
+`ResearchStats`.
+
+**Also on the package entry point**, for callers that want the orchestrators directly rather
+than through the SDK wrapper: `DeepResearchOrchestrator`, `QuickResearchOrchestrator`,
+`HeadlessObserver` (an observer that prints progress to stdout), `ServiceNames`,
+`shutdownManager`, `extractRunStats`, `normalizeUrl`, the config accessors
+`getConfig` / `setConfig` / `resetConfig` / `validateConfig`, and the team-size constants.
+These are lower-level than the SDK functions above and assume you manage the service
+container yourself.
+
 ### Init options
 
 | Option | Description |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.5.2",
+  "version": "1.6.0",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,
@@ -20,8 +20,8 @@
     "dist/thread-worker.mjs",
     "dist/webgpu-probe.mjs",
     "dist/prompts/",
-    "skills/",
-    "!skills/pi-research/scripts/run.ts",
+    "agent-skill/",
+    "!agent-skill/pi-research/scripts/run.ts",
     "!src/.env",
     "!src/.env.local",
     "!src/.env.*.local",
@@ -53,27 +53,15 @@
     "pi-package",
     "pi-extension",
     "pi-coding-agent",
+    "agent-skill",
     "claude-code",
-    "openclaw",
     "codex",
-    "agent-skills",
-    "agent-skill",
-    "skills",
-    "web-search",
+    "openclaw",
     "web-research",
     "deep-research",
-    "ai-research",
-    "research",
+    "web-search",
     "web-scraping",
-    "scraping",
-    "browser-automation",
-    "stealth-browser",
-    "camoufox",
-    "duckduckgo",
-    "knowledge-store",
-    "llm-tools",
-    "web-access",
-    "agent"
+    "knowledge-store"
   ],
   "author": "Lincoln Deen",
   "contributors": [
@@ -105,7 +93,7 @@
     "build": "node scripts/build.cjs all",
     "prepublishOnly": "node scripts/build.cjs all && node scripts/verify-package.cjs manifest",
     "prepare": "node scripts/prepare.cjs",
-    "version": "node scripts/sync-skill-version.cjs && git add skills/pi-research/SKILL.md",
+    "version": "node scripts/sync-skill-version.cjs && git add agent-skill/pi-research/SKILL.md",
     "postinstall": "node scripts/setup.cjs",
     "preuninstall": "node scripts/cleanup.cjs",
     "cleanup": "node scripts/cleanup.cjs",
@@ -130,7 +118,7 @@
     "@kreuzberg/html-to-markdown-node": "^3.5.5",
     "@lancedb/lancedb": "0.37.1",
     "apache-arrow": "21.1.0",
-    "bgutils-js": "^3.2.0",
+    "bgutils-js": "^4.0.3",
     "camoufox-js": "^0.10.2",
     "esbuild": "^0.28.1",
     "impit": "0.13.0",

package/scripts/build.cjs

@@ -12,7 +12,7 @@
  * Targets (dispatch by argv, or build all when no/`all` arg):
  *   worker   -> src/infrastructure/browser/thread-worker.mjs  (browser worker)
  *   cli      -> dist/cli.mjs (+ dist/prompts, dist/thread-worker.mjs, dist/webgpu-probe.mjs)
- *   skill    -> skills/pi-research/scripts/run.mjs  (skill launcher, shebang)
+ *   skill    -> agent-skill/pi-research/scripts/run.mjs  (skill launcher, shebang)
  *
  * Note: `packages: 'external'` already externalizes every bare/`node:` import,
  * so the old `--external:node:*` flag was redundant — output is identical.
@@ -114,10 +114,10 @@ const TARGETS = {
     copyPrompts();
   },
   skill: () => {
-    const entry = p('skills', 'pi-research', 'scripts', 'run.ts');
-    const outfile = p('skills', 'pi-research', 'scripts', 'run.mjs');
+    const entry = p('agent-skill', 'pi-research', 'scripts', 'run.ts');
+    const outfile = p('agent-skill', 'pi-research', 'scripts', 'run.mjs');
     // The published package ships the BUILT run.mjs and deliberately excludes its
-    // run.ts source (package.json "files": "!skills/pi-research/scripts/run.ts"),
+    // run.ts source (package.json "files": "!agent-skill/pi-research/scripts/run.ts"),
     // yet it also ships prepare.cjs + build.cjs — and `prepare` runs `build.cjs all`.
     // On any install that executes prepare against the shipped tree (npm install
     // <dir>, file:, npm link, pi install <dir>) this target had no entry point and