@lincoln504/pi-research 1.3.7 → 1.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/README.md +31 -19
  2. package/dist/cli.mjs +49 -31
  3. package/dist/thread-worker.mjs +37 -22
  4. package/docs/ARCHITECTURE.md +3 -3
  5. package/docs/CHANGELOG.md +56 -0
  6. package/docs/CONFIGURATION.md +4 -3
  7. package/docs/KNOWLEDGE-STORE.md +2 -2
  8. package/docs/PI-EXTENSION.md +2 -2
  9. package/docs/SDK.md +2 -2
  10. package/package.json +2 -2
  11. package/skills/pi-research/SKILL.md +1 -1
  12. package/src/cli.ts +5 -1
  13. package/src/constants.ts +0 -4
  14. package/src/core/pi-version.ts +2 -2
  15. package/src/index.ts +5 -3
  16. package/src/infrastructure/browser/thread-worker.mjs +37 -22
  17. package/src/knowledge/embedder-utils.ts +19 -9
  18. package/src/utils/log-utils.ts +45 -13
  19. package/src/utils/user-agent.ts +26 -15
  20. package/src/web-research/scraper-utils.ts +12 -0
  21. package/src/web-research/web-scraper.ts +6 -1
package/README.md CHANGED
@@ -1,9 +1,9 @@
1
1
  <p align="center">
2
- <img src="https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/banner.jpg" alt="pi-research — free, unlimited web search and deep research for coding agents" width="100%" />
2
+ <img src="https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/banner.jpg" alt="pi-research: free unlimited web search and knowledge store for agents" width="100%" />
3
3
  </p>
4
4
 
5
5
  <p align="center">
6
- <strong>Free, unlimited web search &amp; deep research for coding agents — no API key, no quota</strong>
6
+ <strong>Free unlimited web search &amp; knowledge store for agents</strong>
7
7
  </p>
8
8
 
9
9
  <p align="center">
@@ -22,52 +22,64 @@
22
22
 
23
23
  ---
24
24
 
25
- <a href="https://github.com/Lincoln504/pi-research/actions/workflows/ci.yml"><img alt="Build status" src="https://img.shields.io/github/actions/workflow/status/Lincoln504/pi-research/ci.yml?style=flat-square&branch=main" /></a> <a href="https://www.npmjs.com/package/@lincoln504/pi-research"><img alt="npm version" src="https://img.shields.io/npm/v/@lincoln504/pi-research.svg?style=flat-square" /></a>
25
+ <a href="https://github.com/Lincoln504/pi-research/actions/workflows/ci.yml"><img alt="Build status" src="https://img.shields.io/github/actions/workflow/status/Lincoln504/pi-research/ci.yml?style=flat-square&branch=main" /></a><br />
26
+ <a href="https://www.npmjs.com/package/@lincoln504/pi-research"><img alt="npm version" src="https://img.shields.io/npm/v/@lincoln504/pi-research.svg?style=flat-square" /></a><br />
27
+ <a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-97ca00?style=flat-square" /></a><br />
28
+ <a href="https://github.com/Lincoln504/pi-research"><img alt="GitHub repository" src="https://img.shields.io/badge/GitHub-Lincoln504%2Fpi--research-181717?style=flat-square&logo=github&logoColor=white" /></a>
26
29
 
27
- Search and scraping run locally through a stealth browser — no search provider, no monthly cap. You pay only for the LLM tokens you use.
30
+ Search and scraping run locally through a stealth browser with no search provider and no monthly cap. You pay only for the LLM tokens you use.
28
31
 
29
32
  ### Install
30
33
 
34
+ As a [Pi](https://github.com/earendil-works/pi) Coding Agent extension:
35
+
31
36
  ```bash
32
37
  pi install npm:@lincoln504/pi-research
33
38
  ```
34
39
 
35
- In [pi](https://github.com/earendil-works/pi) it works out of the box, on your session's model and pi's configuration. Standalone use — [agent skill](docs/AGENT-SKILL.md), CLI, or [SDK](docs/SDK.md) — needs a model configured; see [Configuration](docs/CONFIGURATION.md). The first install pulls the stealth browser engine, which takes a few minutes.
40
+ Standalone, as an [agent skill](docs/AGENT-SKILL.md) for Claude Code, Codex, and OpenClaw:
41
+
42
+ ```bash
43
+ npm install -g @lincoln504/pi-research
44
+ pi-research skill install
45
+ ```
46
+
47
+ In pi it works out of the box on your session's model and pi's configuration. Standalone use ([agent skill](docs/AGENT-SKILL.md) or [SDK](docs/SDK.md)) needs a model configured. See [Configuration](docs/CONFIGURATION.md). The first install pulls the stealth browser engine, which takes a few minutes.
36
48
 
37
49
  ### How it works
38
50
 
39
- A research run loops through agent teams: a coordinator plans and opens with a search, researcher agents scrape and read in parallel, and an evaluator decides whether to go another round or synthesize. The result is one cited Markdown report, with findings optionally saved to the knowledge store.
51
+ A research run loops through agent teams: a coordinator plans and opens with a search, researcher agents scrape and read in parallel, and an evaluator decides whether to go another round or synthesize. The result is one cited Markdown report with findings optionally saved to the knowledge store.
40
52
 
41
- Three depth levels — normal, deep, ultra — set the team size and number of rounds. Ask in natural language and the tool picks the right one:
53
+ Three depth levels (normal, deep, ultra) set the team size and number of rounds. Ask in natural language and the tool picks the right one:
42
54
 
43
55
  ![Prompt-driven multi-round research in the pi TUI](https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/02-prompt-research.gif)
44
56
 
45
57
  ### Use cases
46
58
 
47
- - Researching for your [Pi](https://github.com/earendil-works/pi) Coding Agent — plug-and-play, no API key needed.
48
- - Researching from Claude Code, Codex, OpenClaw, or another coding agent, with a cheaper or local model driving the run so it doesn't spend your main agent's budget.
49
- - Keeping a persistent, searchable knowledge store of findings — global or per-project.
50
- - Building agent systems that find and read web content, or populating a dataset of web sources.
59
+ - Researching for your pi. Plug and play with no API key needed.
60
+ - Researching from Claude Code, Codex, OpenClaw, or another coding agent while a cheaper or local model drives the run so it doesn't spend your main agent's budget.
61
+ - Keeping a persistent and searchable knowledge store of findings, scoped globally or per project.
62
+ - Building agent systems that find and read web content or populating a dataset of web sources.
51
63
 
52
64
  ### Why pi-research
53
65
 
54
- - **No quota, no monthly fee.** Most AI-search providers cap free searches and then charge, with results served from *their* index on *their* servers. pi-research searches and scrapes locally — DuckDuckGo via a stealth browser.
55
- - **You own your index.** Every finding is written to a local [LanceDB](https://lancedb.com) knowledge store that seeds future runs, so repeat questions get faster, cheaper answers.
56
- - **Cited reports, not raw dumps.** One synthesized, sourced Markdown report instead of raw web content pasted into the conversation — context-efficient and easy to verify.
66
+ - **No quota and no monthly fee.** Most AI search providers cap free searches and then charge, with results served from their index on their servers. pi-research searches and scrapes locally through DuckDuckGo in a stealth browser.
67
+ - **You own your index.** Every finding is written to a local [LanceDB](https://lancedb.com) knowledge store that seeds future runs, so repeat questions get faster and cheaper answers.
68
+ - **Cited reports rather than raw dumps.** You get one sourced Markdown report instead of raw web content pasted into the conversation. Context efficient and easy to verify.
57
69
  - **Read-only by design.** The research agent cannot run shell commands or write, edit, or delete anything. It can read and search files, so treat it as you would any read-only agent with access to your machine.
58
70
  - **Search a little or a lot.** Depth levels span a quick pass to a large-scale investigation.
59
71
 
60
72
  ### Requirements / limitations
61
73
 
62
74
  - Node.js >= 22.19.0
63
- - An LLM with a 100k+ context window — bring your own key or use a local model
64
- - Internet access on a residential IP — search, scraping, and YouTube transcripts rely on a residential connection; a datacenter/VPS/cloud IP gets bot-blocked
65
- - The [pi runtime](https://github.com/earendil-works/pi) — the pi extension uses the host's copy; the standalone CLI and agent skill install it as a dependency
66
- - Cloudflare and similar systems block scraping on some sites — a run flags pages it can't reach and compensates with a wide pool of search results to cite
75
+ - An LLM with a 100k+ context window (bring your own key or use a local model)
76
+ - Internet access on a residential IP, since search, scraping, and YouTube transcripts get bot-blocked from datacenter/VPS/cloud IPs
77
+ - The pi runtime. The pi extension uses the host's copy while the standalone CLI and agent skill install it as a dependency.
78
+ - Cloudflare and similar systems block scraping on some sites. A run flags pages it can't reach and compensates with a wide pool of search results to cite.
67
79
 
68
80
  ### Channels
69
81
 
70
- npm (`npm:@lincoln504/pi-research`) is the stable channel, kept current with breaking pi changes. A git install is the development channel: latest commits, first to break.
82
+ npm (`npm:@lincoln504/pi-research`) is the stable channel and is kept current with breaking pi changes. A git install is the development channel. It has the latest commits and breaks first.
71
83
 
72
84
  ### License
73
85
 
package/dist/cli.mjs CHANGED
@@ -247,22 +247,36 @@ function redactSecrets(message) {
247
247
  function stripTerminalEscapes(message) {
248
248
  return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
249
249
  }
250
- var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
250
+ var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, B_LEFT, B_RIGHT, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
251
251
  var init_log_utils = __esm({
252
252
  "src/utils/log-utils.ts"() {
253
253
  "use strict";
254
254
  logContextStorage = new AsyncLocalStorage2();
255
255
  MAX_LOG_MESSAGE_LENGTH = 1e4;
256
256
  ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
257
- URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
258
- COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
259
- SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
260
- KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
261
- JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
262
- BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
263
- BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
264
- PROVIDER_TOKEN_PATTERN = /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
265
- LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
257
+ URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]{0,31}:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
258
+ B_LEFT = "(?<![A-Za-z0-9])";
259
+ B_RIGHT = "(?![A-Za-z0-9])";
260
+ COOKIE_HEADER_PATTERN = new RegExp(
261
+ `${B_LEFT}(set[_-]?cookie|cookie)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\r\\n]+)`,
262
+ "gi"
263
+ );
264
+ SENSITIVE_KV_PATTERN = new RegExp(
265
+ `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
266
+ "gi"
267
+ );
268
+ KNOWN_TOKEN_PATTERN = new RegExp(
269
+ `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
270
+ "g"
271
+ );
272
+ JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, "g");
273
+ BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
274
+ BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
275
+ PROVIDER_TOKEN_PATTERN = new RegExp(
276
+ `${B_LEFT}(AIza[0-9A-Za-z_-]{35}|ya29\\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})${B_RIGHT}`,
277
+ "g"
278
+ );
279
+ LONG_HEX_SECRET_PATTERN = new RegExp(`${B_LEFT}[0-9a-fA-F]{32,}${B_RIGHT}`, "g");
266
280
  CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
267
281
  REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
268
282
  TOKEN_CHAR_PATTERN = /[A-Za-z0-9_\-./+=]/;
@@ -5131,14 +5145,16 @@ function unregisterGlobalEmbedder() {
5131
5145
  async function initializeDawnWebGPU() {
5132
5146
  if (dawnInitialized) return true;
5133
5147
  try {
5134
- const { create, globals } = await import("webgpu");
5135
- Object.assign(globalThis, globals);
5136
- Object.defineProperty(globalThis, "navigator", {
5137
- value: { gpu: create([]) },
5138
- writable: true,
5139
- configurable: true
5148
+ const adapter = await logger.runCapturingStderr(async () => {
5149
+ const { create, globals } = await import("webgpu");
5150
+ Object.assign(globalThis, globals);
5151
+ Object.defineProperty(globalThis, "navigator", {
5152
+ value: { gpu: create([]) },
5153
+ writable: true,
5154
+ configurable: true
5155
+ });
5156
+ return await globalThis.navigator.gpu.requestAdapter({ powerPreference: "high-performance" });
5140
5157
  });
5141
- const adapter = await globalThis.navigator.gpu.requestAdapter({ powerPreference: "high-performance" });
5142
5158
  if (!adapter) {
5143
5159
  logger.warn("[embedder] No WebGPU adapter available \u2014 using CPU.");
5144
5160
  return false;
@@ -16808,21 +16824,21 @@ init_error_tracker();
16808
16824
 
16809
16825
  // src/utils/user-agent.ts
16810
16826
  var USER_AGENTS = [
16811
- // Chrome on Windows (updated 2025)
16812
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
16813
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
16827
+ // Chrome on Windows
16828
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
16829
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
16814
16830
  // Chrome on macOS
16815
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
16816
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
16831
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
16832
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
16817
16833
  // Chrome on Linux
16818
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
16819
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
16820
- // Firefox on Windows (updated 2025)
16821
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
16822
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0",
16834
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
16835
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
16836
+ // Firefox on Windows
16837
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0",
16838
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0",
16823
16839
  // Firefox on macOS
16824
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0",
16825
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:138.0) Gecko/20100101 Firefox/138.0"
16840
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0",
16841
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0"
16826
16842
  ];
16827
16843
 
16828
16844
  // src/web-research/scraper-types.ts
@@ -17068,6 +17084,7 @@ function isPrivateIpv6(ip) {
17068
17084
  if (normalized.startsWith("2002:")) return true;
17069
17085
  if (/^2001:0{0,4}:/.test(normalized) || normalized.startsWith("2001::")) return true;
17070
17086
  if (normalized.startsWith("64:ff9b:")) return true;
17087
+ if (/^::(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f]{1,4}(?::[0-9a-f]{1,4})?)$/.test(normalized)) return true;
17071
17088
  if (normalized === "::" || normalized === "0:0:0:0:0:0:0:0") return true;
17072
17089
  return false;
17073
17090
  }
@@ -17296,12 +17313,13 @@ async function scrapeWithFetch(url, signal) {
17296
17313
  const MAX_REDIRECTS = 10;
17297
17314
  let currentUrl = url;
17298
17315
  let response;
17316
+ const userAgent = getRandomUserAgent();
17299
17317
  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
17300
17318
  response = await fetch(currentUrl, {
17301
17319
  signal: controller.signal,
17302
17320
  redirect: "manual",
17303
17321
  headers: {
17304
- "User-Agent": getRandomUserAgent(),
17322
+ "User-Agent": userAgent,
17305
17323
  "Accept": "text/html,application/xhtml+xml,application/pdf,*/*;q=0.8"
17306
17324
  },
17307
17325
  // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.
@@ -25674,7 +25692,7 @@ function parseArgs(argv) {
25674
25692
  }
25675
25693
  if (a === "--depth") {
25676
25694
  const v = rest[++i];
25677
- const n = v === void 0 ? NaN : parseInt(v, 10);
25695
+ const n = v !== void 0 && /^\d+$/.test(v.trim()) ? Number(v.trim()) : NaN;
25678
25696
  if (isNaN(n) || n < 0 || n > 3) throw new UsageError("--depth must be an integer 0\u20133.");
25679
25697
  depth = n;
25680
25698
  } else if (a === "--model") {
package/dist/thread-worker.mjs CHANGED
@@ -77,22 +77,36 @@ function redactSecrets(message) {
77
77
  function stripTerminalEscapes(message) {
78
78
  return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
79
79
  }
80
- var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
80
+ var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, B_LEFT, B_RIGHT, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
81
81
  var init_log_utils = __esm({
82
82
  "src/utils/log-utils.ts"() {
83
83
  "use strict";
84
84
  logContextStorage = new AsyncLocalStorage();
85
85
  MAX_LOG_MESSAGE_LENGTH = 1e4;
86
86
  ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
87
- URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
88
- COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
89
- SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
90
- KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
91
- JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
92
- BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
93
- BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
94
- PROVIDER_TOKEN_PATTERN = /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
95
- LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
87
+ URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]{0,31}:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
88
+ B_LEFT = "(?<![A-Za-z0-9])";
89
+ B_RIGHT = "(?![A-Za-z0-9])";
90
+ COOKIE_HEADER_PATTERN = new RegExp(
91
+ `${B_LEFT}(set[_-]?cookie|cookie)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\r\\n]+)`,
92
+ "gi"
93
+ );
94
+ SENSITIVE_KV_PATTERN = new RegExp(
95
+ `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
96
+ "gi"
97
+ );
98
+ KNOWN_TOKEN_PATTERN = new RegExp(
99
+ `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
100
+ "g"
101
+ );
102
+ JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, "g");
103
+ BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
104
+ BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
105
+ PROVIDER_TOKEN_PATTERN = new RegExp(
106
+ `${B_LEFT}(AIza[0-9A-Za-z_-]{35}|ya29\\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})${B_RIGHT}`,
107
+ "g"
108
+ );
109
+ LONG_HEX_SECRET_PATTERN = new RegExp(`${B_LEFT}[0-9a-fA-F]{32,}${B_RIGHT}`, "g");
96
110
  CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
97
111
  REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
98
112
  TOKEN_CHAR_PATTERN = /[A-Za-z0-9_\-./+=]/;
@@ -110,21 +124,21 @@ var init_user_agent = __esm({
110
124
  "src/utils/user-agent.ts"() {
111
125
  "use strict";
112
126
  USER_AGENTS = [
113
- // Chrome on Windows (updated 2025)
114
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
115
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
127
+ // Chrome on Windows
128
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
129
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
116
130
  // Chrome on macOS
117
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
118
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
131
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
132
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
119
133
  // Chrome on Linux
120
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
121
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
122
- // Firefox on Windows (updated 2025)
123
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
124
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0",
134
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
135
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
136
+ // Firefox on Windows
137
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0",
138
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0",
125
139
  // Firefox on macOS
126
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0",
127
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:138.0) Gecko/20100101 Firefox/138.0"
140
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0",
141
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0"
128
142
  ];
129
143
  }
130
144
  });
@@ -1678,6 +1692,7 @@ function isPrivateIpv6(ip) {
1678
1692
  if (normalized.startsWith("2002:")) return true;
1679
1693
  if (/^2001:0{0,4}:/.test(normalized) || normalized.startsWith("2001::")) return true;
1680
1694
  if (normalized.startsWith("64:ff9b:")) return true;
1695
+ if (/^::(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f]{1,4}(?::[0-9a-f]{1,4})?)$/.test(normalized)) return true;
1681
1696
  if (normalized === "::" || normalized === "0:0:0:0:0:0:0:0") return true;
1682
1697
  return false;
1683
1698
  }
package/docs/ARCHITECTURE.md CHANGED
@@ -3,8 +3,8 @@
3
3
  pi-research is a pi TUI extension for multi-agent web research. It runs inside the pi
4
4
  process, registers its tools and commands, and manages its own browser worker pool,
5
5
  service registry, and local knowledge store. One engine backs every front-end: besides
6
- the pi extension it is exposed as a standalone CLI, a portable agent skill — the same
7
- skill any skills-aware host runs, including OpenClaw — and a programmatic SDK
6
+ the pi extension it is exposed as a standalone CLI, a portable agent skill (the same
7
+ skill any skills-aware host runs, including OpenClaw) and a programmatic SDK
8
8
  (`src/sdk.ts`).
9
9
 
10
10
  ```
@@ -318,7 +318,7 @@ other). This sits above `@lancedb/lancedb` 0.29's declared Arrow peer ceiling
318
318
  whenever `@lancedb/lancedb` is upgraded.
319
319
 
320
320
  Pinned validation library — `typebox` is pinned to the exact version the pi host packages
321
- depend on (`@earendil-works/pi-ai` / `@earendil-works/pi-coding-agent` 0.83.0 pin `1.3.7`). Every
321
+ depend on (`@earendil-works/pi-ai` / `@earendil-works/pi-coding-agent` 0.84.2 pin `1.3.7`). Every
322
322
  tool's parameter schema is built with TypeBox here and handed across the boundary to pi's
323
323
  tool system, so the two must agree on `Value.Check`/`Convert` semantics. A floating `^1.1.38`
324
324
  range let a fresh consumer install resolve pi-research to a newer TypeBox than pi's, shipping
package/docs/CHANGELOG.md CHANGED
@@ -5,6 +5,62 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.3.9] - 2026-08-14
9
+
10
+ A whole-tree audit sweep. Six defects, each found by looking at what the code
11
+ actually does at runtime rather than at what the last review changed: two of them
12
+ (the redaction boundary and the SSRF literal range) are further instances of bug
13
+ classes earlier releases fixed elsewhere but did not fully close, and one — the
14
+ quadratic backtracking in the redactor — surfaced only because the boundary fix
15
+ was benchmarked before being accepted. Every fix below carries a regression test
16
+ confirmed, by a local revert-and-rerun, to fail against the pre-fix code.
17
+
18
+ ### Fixed
19
+
20
+ - **Redacting one large log message burned ~1.4 seconds of CPU.** `URL_CREDENTIALS_PATTERN` matched the URL scheme with an unbounded `[a-z0-9+.-]*`, so at every position inside a long alphanumeric run the engine consumed the rest of the input and then backtracked one character at a time looking for `://` — quadratic in the input length. `redactSecrets()` runs on **every** log message and deliberately scans a 40_000-character window, and the content it scans is largely untrusted: scraped pages and provider error bodies. A single page carrying one long unbroken token was enough, and the cost lands on the main thread. Bounding the scheme (RFC 3986 schemes are short; none approaches 32 characters) makes each start position O(32) instead of O(n) for identical matches — measured 1439 ms → 6.7 ms on a hostile 40k input, ~215x. Three pre-existing redaction tests dropped from 394/107/91 ms to 4/2/2 ms as a side effect, which is how long the cost had been sitting in plain sight. A regression test now pins the bound ~100x below the pre-fix cost, loose enough not to flake on a slow CI runner.
21
+ - **Log redaction masked `api_key=` but not `PI_RESEARCH_API_KEY=`.** `\b` is not a word boundary between `_` and a letter — both are word characters — so every secret whose key name carried an underscore-joined prefix passed through the redactor untouched and was written to the diagnostic log in the clear. That is the single most common way a credential is ever written down — this tool's own `PI_RESEARCH_API_KEY`, and every `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` / `DB_PASSWORD` / `MY_SECRET` beside it — and it reaches the log by more than one route: `redactSecrets()` is the last-resort filter over *arbitrary* content, which for a research tool includes scraped pages and provider error bodies. The existing `PI_BROWSER_AUTH_SECRET=` coverage passed only because a 64-char run of `a` is also valid hex, so the long-hex pattern caught it and the key-name gap stayed hidden. All secret-name and opaque-token patterns now treat any non-alphanumeric character as the boundary, so a prefixed name matches while a name merely *containing* the word (`monkey=`, `keyboard=`, `passwordless=`) still does not.
22
+ - **SSRF: the deprecated IPv4-compatible `::/96` range was accepted.** `http://[::127.0.0.1]/` normalizes to `[::7f00:1]`, which matched none of the guards: it is not `::1`, carries no `fc`/`fd`/`fe8x` prefix, has no `::ffff:` prefix for the mapped-IPv4 decoder to read, and is none of the 6to4/Teredo/NAT64 tunnel prefixes that are already blocked whole-range. It was the last IPv4-in-IPv6 form still open, and whether it reaches the embedded address depends on the OS, an intervening proxy, and NAT64 translation — the same "depends on the path" property that made those other ranges whole-range blocks. Now blocked structurally (top 96 bits zero), which is why it cannot swallow the mapped `::ffff:0:0/96` range next to it.
23
+ - **Dawn printed WebGPU warnings onto the user's terminal.** onnxruntime-node's bundled Dawn writes its limit-clamping notices ("`maxDynamicUniformBuffersPerPipelineLayout artificially reduced from 1000000 to 16`") to FD 2 from native code the moment a WebGPU instance is created, so a plain `pi-research knowledge …` run emitted four of them onto the terminal — and for the agent skill they landed in the stderr an agent is instructed to read for *errors*. `loadPipelineWithTimeout` already builds its pipeline inside the FD-level stderr capture for exactly this reason; the adapter probe in `initializeDawnWebGPU` is the other native entry point and was missing it. Confirmed gone on a real run, not just in the test.
24
+ - **`--depth` accepted a value that only started with a valid integer.** `parseInt` scans a leading prefix, so `--depth 2x` ran depth 2 and `--depth 2.9` ran depth 2, silently, while the error text promised "an integer 0–3". Depth is the knob that decides how long a run takes and what it costs, so a typo picking a different one is not a harmless clamp. The whole token is now matched.
25
+ - **The fetch scrape path rotated its User-Agent per redirect hop.** The UA was picked inside the manual redirect loop, so one scrape could present as Chrome on Windows for the first hop and Firefox on macOS for the next — something no browser does, which turned the rotation meant to blend in into a distinctive signal, on exactly the multi-hop requests (consent gates, tracking redirectors) where bot detection is most attentive. One UA now covers the whole chain.
26
+
27
+ ### Changed
28
+
29
+ - **User-Agent pool refreshed to current stable browsers.** It was pinned at Chrome 137/136 and Firefox 139/138 — around fourteen major versions behind on both, since each ships roughly every four weeks. A UA that stale is itself a bot signal, which is the thing rotating it exists to avoid. Now Chrome 151/150 and Firefox 153/152, verified against Chrome Releases (151 stable, 2026-07-28) and the Firefox release notes (153 stable, 2026-08-11), with the shelf life and the refresh procedure written down at the definition and a test guarding the *shape* of a future refresh (a `rv:`/`Firefox/` major mismatch, a missing Chrome minor triplet, an over-wide version spread) rather than a version floor that would go stale on the browsers' schedule.
30
+ - **pi 0.84.2.** Read the full (not grepped) 0.84.2 changelog: no breaking changes section, and its one dependency-visible change — the Mistral SDK transport replaced by a native HTTP stream — shows up here only as `@mistralai/mistralai`, `zod`, and `zod-to-json-schema` dropping out of the lockfile. `PI_TESTED_MAX_VERSION` moved to 0.84.2 to record what was actually exercised (full unit + integration suites, plus a live knowledge-store run). The supported floor is unchanged at 0.84.0.
31
+ - **Release workflow serializes.** Added the `concurrency: release` group the workflow's own TODO called for, with `cancel-in-progress: false` so a release already mid-publish is never cancelled by a newer tag. Closes one of the two documented hardening TODOs; SHA-pinning the actions remains open.
32
+ - **README wording.** Tagline and alt text read "for agents", the install label reads "As a Pi Coding Agent extension" with only "Pi" linked, the standalone list drops the CLI and links the agent-skill doc, and the italics and surplus links in the use-case and requirements bullets are gone.
33
+ - Removed the dead `REQUEST_DELAY_MS_NVD` / `REQUEST_DELAY_MS_OTHER` constants (no reader anywhere in the tree; the security clients carry their own throttles).
34
+
35
+ ### Documentation
36
+
37
+ - **`PI_RESEARCH_ALLOW_LOOPBACK_SCRAPE` was documented as a wider hole than it is.** The table described it as permitting "loopback/private/internal addresses, bypassing SSRF protection". The flag is deliberately scoped to loopback only — link-local `169.254.0.0/16` (cloud metadata) and RFC1918 LAN ranges stay blocked with it set, at request time and at connect time, and a test pins that. Corrected, since an inaccurate description of a security control is worse than none.
38
+ - Documented `PI_RESEARCH_STRICT_SETUP` (read by `scripts/setup.cjs`; makes a failed browser download fail the install instead of deferring it to first use) in both `docs/CONFIGURATION.md` and `.env.example` — it was the only environment variable in the tree missing from either, and `.env.example` states outright that it is the exhaustive reference.
39
+ - Corrected two stale in-code comments that still named `>=0.80.8` as the declared `@earendil-works/*` range after the floor moved to 0.84.0 in 1.3.3, and the pi version cited in the ARCHITECTURE typebox pin rationale (0.84.1 → 0.84.2; the pinned typebox is still 1.3.7).
40
+
41
+ ### Verified
42
+
43
+ - Full unit suite, the cross-group integration suite against a real Camoufox browser, lint, both type-checks, the architectural dependency rules, and the production audit gate — all green. Live end-to-end checks: all four security databases (NVD, CISA KEV, GitHub Advisories, OSV) answered with the expected shapes, and a knowledge-store run exercised the embedder, LanceDB, and the synthesis path. A live `research` run could not complete — the configured OpenRouter account is out of credits — but the engine reported that correctly: exit 70, the provider's own 402 surfaced as the cause, and no report fabricated from nothing.
44
+ - The single allowlisted advisory (GHSA-f88m-g3jw-g9cj, `sharp` via `@huggingface/transformers`, unreachable image path) still has no upstream fix; no new advisories. No open CodeQL alerts, issues, or pull requests.
45
+ - Swept the rest of the tree for the same backtracking shape rather than fixing only the instance that was measured. The remaining candidates are lazy or anchored, and the regexes that actually run over scraped content (`IMAGE_LINK_PATTERN`, `MARKDOWN_IMAGE_PATTERN`, the blank-line collapse) stay under 20 ms on 500_000-character hostile inputs — unbalanced brackets, unterminated parens, single-character runs — at the scrape cap. The redactor was the only instance.
46
+ - Environment-variable coverage is now exhaustive in both directions: every `PI_RESEARCH_*` name read anywhere in `src/` or `scripts/` appears in `docs/CONFIGURATION.md` and `.env.example`, and every documented name is read somewhere. The one deliberate omission is the `PI_RESEARCH_KNOWLEDGE_STORE_CACHE_TTL_DAYS` read-time alias, which exists only for backward compatibility and should not be advertised over the canonical `PI_RESEARCH_CACHE_TTL_DAYS`.
47
+ - Considered and rejected: the chunker's per-iteration `text.slice(0, start).match(/```/g)` is also quadratic in document length, but measured at 22 ms for a 500k-character document — the scrape cap — which does not justify changing code on the storage hot path. Measuring is what separated it from the redactor case above, where the same shape cost 1.4 s.
48
+
49
+ ## [1.3.8] - 2026-08-13
50
+
51
+ Presentation and dependency maintenance only. No engine changes.
52
+
53
+ ### Changed
54
+
55
+ - **README refined.** New tagline ("Free unlimited web search & knowledge store for coding agents"), MIT license and GitHub repository badges added with all four badges stacked vertically, both install paths (pi extension and standalone agent skill) shown up top, and a punctuation pass removing em dashes and surplus commas and semicolons from README and the npm description. The punctuation pass now also covers the intro paragraphs of the docs/ pages.
56
+ - **npm description compacted.** The no-search-API-key and no-quotas clause is covered by "free unlimited" and is gone.
57
+ - **docs/ARCHITECTURE.md** typebox pin rationale updated to cite the current pi host version (0.84.1, which still pins typebox 1.3.7).
58
+ - **In-range dependency refresh.** esbuild 0.28.2, undici 8.10.0, eslint 10.8.1, typescript-eslint 8.67.0. Deliberate exact pins (playwright-core 1.60.0, camoufox-js 0.10.x, lancedb 0.29.0, apache-arrow 21.1.0, transformers 4.2.0) unchanged per their documented rationale.
59
+
60
+ ### Security
61
+
62
+ - Production audit gate re-verified. The single allowlisted advisory (GHSA-f88m-g3jw-g9cj, sharp via transformers, unreachable image path) still has no upstream fix; no new advisories.
63
+
8
64
  ## [1.3.7] - 2026-08-11
9
65
 
10
66
  Discovery and packaging metadata only — no engine changes.
package/docs/CONFIGURATION.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ## Configuration
2
2
 
3
- Every front-end (the pi extension, the standalone CLI / agent skill — the surface
4
- OpenClaw and other skills-aware hosts run — and the SDK) shares one configuration
3
+ Every front-end (the pi extension, the standalone CLI / agent skill that OpenClaw
4
+ and other skills-aware hosts run, and the SDK) shares one configuration
5
5
  model. This document covers the
6
6
  settings exposed in the `/research-config` TUI first, then the complete
7
7
  environment-variable reference, and finally how the configuration layers resolve.
@@ -159,6 +159,7 @@ Diagnostics & platform
159
159
  | `PLAYWRIGHT_BROWSERS_PATH` | _(user cache)_ | Standard Playwright variable, honoured here too: overrides where the camoufox binary is stored and looked up, and is passed through to browser workers. |
160
160
  | `XDG_CACHE_HOME` | `~/.cache` | Standard XDG variable. When set, every `~/.cache/pi-research/...` path below is rooted at `$XDG_CACHE_HOME/pi-research/...` instead. |
161
161
  | `PLAYWRIGHT_INSTALL_DEPS` | _(unset)_ | Linux only. Set `true` during `npm install` to also install system libraries via `npx playwright install-deps` (same as `npm run install:system-deps`). |
162
+ | `PI_RESEARCH_STRICT_SETUP` | _(unset)_ | Read by the bundled `scripts/setup.cjs` during `npm install`. Set `1`/`true` to make a failed browser download fail the install instead of deferring it to first use. |
162
163
  | `PI_RESEARCH_CONFIG_DIR_NAME` | `.pi` | Override the host config-directory name under your home dir (advanced; e.g. set to share another harness's config root). |
163
164
 
164
165
  Testing only — never enable in production
@@ -168,7 +169,7 @@ Testing only — never enable in production
168
169
  | `PI_RESEARCH_MOCK_SEARCH` | Return fabricated search results instead of real web data. |
169
170
  | `PI_RESEARCH_MOCK_SCRAPE` | Return fabricated scrape results instead of real page content. |
170
171
  | `PI_RESEARCH_FORCE_READY` | Bypass readiness checks and run even when critical services failed to initialize. **pi extension only** — the CLI, agent skill and SDK do not consult it. |
171
- | `PI_RESEARCH_ALLOW_LOOPBACK_SCRAPE` | Permit scraping loopback/private/internal addresses, bypassing SSRF protection. |
172
+ | `PI_RESEARCH_ALLOW_LOOPBACK_SCRAPE` | Permit scraping **loopback** addresses (`127.0.0.0/8`, `::1`, `*.localhost`, `::ffff:127.x`) so integration tests can drive the real browser and scrape pipeline against a local server. Deliberately scoped to loopback only: link-local `169.254.0.0/16` (cloud metadata) and RFC1918 LAN ranges stay blocked even with this set, at request time and at connect time. |
172
173
 
173
174
  ### How configuration is layered
174
175
 
package/docs/KNOWLEDGE-STORE.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ## Knowledge Store
2
2
 
3
- The knowledge store is a local vector database of past research findings. It is used
4
- in two distinct ways, and it is an optional cache — research works without it:
3
+ The knowledge store is a local vector database of past research findings. It is an
4
+ optional cache (research works without it) and is used in two distinct ways:
5
5
 
6
6
  - **Knowledge-first answering (advisory).** The `research_knowledge_search` tool
7
7
  answers a repeat or overlapping question directly from stored results. The agent is
package/docs/PI-EXTENSION.md CHANGED
@@ -1,8 +1,8 @@
1
1
  ## Pi Extension
2
2
 
3
3
  pi-research integrates as a [pi](https://github.com/earendil-works/pi)
4
- extension (`src/index.ts`) — a multi-agent web research engine with a real-time
5
- TUI, registered directly in the pi process.
4
+ extension (`src/index.ts`), a multi-agent web research engine with a real-time
5
+ TUI registered directly in the pi process.
6
6
 
7
7
  ### Usage
8
8
 
package/docs/SDK.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ## SDK
2
2
 
3
- The programmatic SDK for scripts, CI, and custom tooling. For configuration — the
4
- layering model, the TUI settings, and every environment variable — see
3
+ The programmatic SDK for scripts, CI, and custom tooling. For configuration (the
4
+ layering model, the TUI settings, and every environment variable) see
5
5
  [CONFIGURATION.md](CONFIGURATION.md).
6
6
 
7
7
  ### Install
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@lincoln504/pi-research",
3
- "version": "1.3.7",
4
- "description": "Free, unlimited web search and deep research for the Pi coding agent — no search API key, no quotas. Local knowledge store; also works from Claude Code, Codex, and OpenClaw.",
3
+ "version": "1.3.9",
4
+ "description": "Free unlimited web search and deep research for the Pi coding agent. Local knowledge store. Also works from Claude Code, Codex, and OpenClaw.",
5
5
  "type": "module",
6
6
  "private": false,
7
7
  "main": "./src/index.ts",
package/skills/pi-research/SKILL.md CHANGED
@@ -2,7 +2,7 @@
2
2
  name: pi-research
3
3
  description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
4
4
  license: MIT
5
- metadata: { "author": "Lincoln Deen", "version": "1.3.7", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
5
+ metadata: { "author": "Lincoln Deen", "version": "1.3.9", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
6
6
  allowed-tools: Bash(node:*)
7
7
  ---
8
8
 
package/src/cli.ts CHANGED
@@ -1360,7 +1360,11 @@ export function parseArgs(argv: string[]): ParsedArgs {
1360
1360
  }
1361
1361
  if (a === '--depth') {
1362
1362
  const v = rest[++i];
1363
- const n = v === undefined ? NaN : parseInt(v, 10);
1363
+ // Match the WHOLE token, not parseInt's leading-prefix scan: parseInt
1364
+ // reads '2x' as 2 and '2.9' as 2, so a typo silently ran a different
1365
+ // depth — the one knob that decides how long a run takes and what it
1366
+ // costs — while the error text promised "an integer 0–3".
1367
+ const n = v !== undefined && /^\d+$/.test(v.trim()) ? Number(v.trim()) : NaN;
1364
1368
  if (isNaN(n) || n < 0 || n > 3) throw new UsageError('--depth must be an integer 0–3.');
1365
1369
  depth = n;
1366
1370
  } else if (a === '--model') {
package/src/constants.ts CHANGED
@@ -8,10 +8,6 @@ import { getConfig, type Config } from './config.ts';
8
8
 
9
9
  // ==================== Time Constants ====================
10
10
 
11
- /** Delay between API requests in milliseconds */
12
- export const REQUEST_DELAY_MS_NVD = 6000;
13
- export const REQUEST_DELAY_MS_OTHER = 1000;
14
-
15
11
  /** Duration (ms) for a green (success) flash on a researcher column */
16
12
  export const FLASH_GREEN_DURATION_MS = 400;
17
13
 
package/src/core/pi-version.ts CHANGED
@@ -7,7 +7,7 @@
7
7
  *
8
8
  * - As a pi EXTENSION, the host supplies `@earendil-works/*`. Our package.json
9
9
  * range never constrains it, so an in-process check is the only enforcement.
10
- * - As a standalone CLI/SDK, the range is `>=0.80.8 <1` and published tarballs
10
+ * - As a standalone CLI/SDK, the range is `>=0.84.0 <1` and published tarballs
11
11
  * carry no lockfile, so every fresh install resolves the newest 0.x at that
12
12
  * instant. pi is pre-1.0: a minor bump may break anything under semver, and
13
13
  * already has (0.83.0 extended the ResourceLoader contract).
@@ -83,7 +83,7 @@ export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 84, patch: 0 };
83
83
  * Compared on MAJOR.MINOR only: a patch bump within a tested line is not a new
84
84
  * surface, and warning on it would be noise.
85
85
  */
86
- export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 84, patch: 1 };
86
+ export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 84, patch: 2 };
87
87
 
88
88
  export type PiCompatibilityLevel = 'ok' | 'unparseable' | 'too-old' | 'untested';
89
89
 
package/src/index.ts CHANGED
@@ -91,12 +91,14 @@ function extractResultText(result: AgentToolResult<unknown>): string {
91
91
  * Pi Research Extension
92
92
  */
93
93
  export default async function (pi: ExtensionAPI) {
94
- // Runtime version check — must match the @earendil-works/* dependency minimum (>=0.80.8).
95
- // 0.80.8 is the floor because pi 0.80.8 introduced ModelRuntime and removed
94
+ // Runtime version check — must match the @earendil-works/* dependency minimum (>=0.84.0).
95
+ // The floor is 0.84.0 because setRuntimeApiKey() lost its `allowNetwork` option there
96
+ // and only 0.84.0+ hardcodes the equivalent guard internally (see pi-version.ts). The
97
+ // APIs it rests on arrived in 0.80.8, which introduced ModelRuntime and removed
96
98
  // AuthStorage/ModelRegistry.create(); buildModelRegistry (model-registry-factory.ts)
97
99
  // unconditionally calls ModelRuntime.create(), and createAgentSession() is invoked
98
100
  // without the removed `modelRegistry` option (researcher.ts), relying on the
99
- // 0.80.8+ host auto-building its ModelRuntime. This in-host check is the real
101
+ // host auto-building its ModelRuntime. This in-host check is the real
100
102
  // enforcement point: when running as an extension the host's version is NOT
101
103
  // constrained by npm's resolution of our own package.json range.
102
104
  // Two thresholds, not one — see src/core/pi-version.ts. Below the floor the APIs
package/src/infrastructure/browser/thread-worker.mjs CHANGED
@@ -77,22 +77,36 @@ function redactSecrets(message) {
77
77
  function stripTerminalEscapes(message) {
78
78
  return message.replace(TERMINAL_ESCAPE_PATTERN, "").replace(CONTROL_CHARS_PATTERN, " ");
79
79
  }
80
- var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
80
+ var logContextStorage, MAX_LOG_MESSAGE_LENGTH, ANSI_PATTERN, URL_CREDENTIALS_PATTERN, B_LEFT, B_RIGHT, COOKIE_HEADER_PATTERN, SENSITIVE_KV_PATTERN, KNOWN_TOKEN_PATTERN, JWT_PATTERN, BASIC_AUTH_PATTERN, BEARER_AUTH_PATTERN, PROVIDER_TOKEN_PATTERN, LONG_HEX_SECRET_PATTERN, CONTROL_CHARS_PATTERN, REDACT_SCAN_LENGTH, TOKEN_CHAR_PATTERN, REDACT_SCAN_EXTENSION_MAX, TERMINAL_ESCAPE_PATTERN;
81
81
  var init_log_utils = __esm({
82
82
  "src/utils/log-utils.ts"() {
83
83
  "use strict";
84
84
  logContextStorage = new AsyncLocalStorage();
85
85
  MAX_LOG_MESSAGE_LENGTH = 1e4;
86
86
  ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
87
- URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
88
- COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
89
- SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
90
- KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
91
- JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
92
- BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
93
- BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
94
- PROVIDER_TOKEN_PATTERN = /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
95
- LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
87
+ URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]{0,31}:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
88
+ B_LEFT = "(?<![A-Za-z0-9])";
89
+ B_RIGHT = "(?![A-Za-z0-9])";
90
+ COOKIE_HEADER_PATTERN = new RegExp(
91
+ `${B_LEFT}(set[_-]?cookie|cookie)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\r\\n]+)`,
92
+ "gi"
93
+ );
94
+ SENSITIVE_KV_PATTERN = new RegExp(
95
+ `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
96
+ "gi"
97
+ );
98
+ KNOWN_TOKEN_PATTERN = new RegExp(
99
+ `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
100
+ "g"
101
+ );
102
+ JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, "g");
103
+ BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
104
+ BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
105
+ PROVIDER_TOKEN_PATTERN = new RegExp(
106
+ `${B_LEFT}(AIza[0-9A-Za-z_-]{35}|ya29\\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})${B_RIGHT}`,
107
+ "g"
108
+ );
109
+ LONG_HEX_SECRET_PATTERN = new RegExp(`${B_LEFT}[0-9a-fA-F]{32,}${B_RIGHT}`, "g");
96
110
  CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
97
111
  REDACT_SCAN_LENGTH = MAX_LOG_MESSAGE_LENGTH * 4;
98
112
  TOKEN_CHAR_PATTERN = /[A-Za-z0-9_\-./+=]/;
@@ -110,21 +124,21 @@ var init_user_agent = __esm({
110
124
  "src/utils/user-agent.ts"() {
111
125
  "use strict";
112
126
  USER_AGENTS = [
113
- // Chrome on Windows (updated 2025)
114
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
115
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
127
+ // Chrome on Windows
128
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
129
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
116
130
  // Chrome on macOS
117
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
118
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
131
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
132
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
119
133
  // Chrome on Linux
120
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
121
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
122
- // Firefox on Windows (updated 2025)
123
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
124
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0",
134
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
135
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
136
+ // Firefox on Windows
137
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0",
138
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0",
125
139
  // Firefox on macOS
126
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0",
127
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:138.0) Gecko/20100101 Firefox/138.0"
140
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0",
141
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0"
128
142
  ];
129
143
  }
130
144
  });
@@ -1678,6 +1692,7 @@ function isPrivateIpv6(ip) {
1678
1692
  if (normalized.startsWith("2002:")) return true;
1679
1693
  if (/^2001:0{0,4}:/.test(normalized) || normalized.startsWith("2001::")) return true;
1680
1694
  if (normalized.startsWith("64:ff9b:")) return true;
1695
+ if (/^::(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f]{1,4}(?::[0-9a-f]{1,4})?)$/.test(normalized)) return true;
1681
1696
  if (normalized === "::" || normalized === "0:0:0:0:0:0:0:0") return true;
1682
1697
  return false;
1683
1698
  }
package/src/knowledge/embedder-utils.ts CHANGED
@@ -254,16 +254,26 @@ export async function initializeDawnWebGPU(): Promise<boolean> {
254
254
  if (dawnInitialized) return true;
255
255
 
256
256
  try {
257
- // @ts-ignore — optional dependency
258
- const { create, globals } = await import('webgpu');
259
- Object.assign(globalThis, globals);
260
- // Use Object.defineProperty — Node.js defines navigator as a configurable getter
261
- Object.defineProperty(globalThis, 'navigator', {
262
- value: { gpu: create([]) },
263
- writable: true,
264
- configurable: true,
257
+ // Dawn writes its limit-clamping notices ("maxDynamicUniformBuffersPer-
258
+ // PipelineLayout artificially reduced from 1000000 to 16 …") straight to
259
+ // FD 2 from native code the moment create() spins up an instance, so they
260
+ // land on the user's terminal — and, for the agent skill, in the stderr an
261
+ // agent is told to read for ERRORS. loadPipelineWithTimeout already runs the
262
+ // pipeline construction inside this same FD-level capture for exactly this
263
+ // reason; the adapter probe below is the other native entry point and was
264
+ // missing it, which is why a plain `knowledge` run still printed them.
265
+ const adapter = await logger.runCapturingStderr(async () => {
266
+ // @ts-ignore — optional dependency
267
+ const { create, globals } = await import('webgpu');
268
+ Object.assign(globalThis, globals);
269
+ // Use Object.defineProperty — Node.js defines navigator as a configurable getter
270
+ Object.defineProperty(globalThis, 'navigator', {
271
+ value: { gpu: create([]) },
272
+ writable: true,
273
+ configurable: true,
274
+ });
275
+ return await (globalThis as any).navigator.gpu.requestAdapter({ powerPreference: 'high-performance' });
265
276
  });
266
- const adapter = await (globalThis as any).navigator.gpu.requestAdapter({ powerPreference: 'high-performance' });
267
277
 
268
278
  if (!adapter) {
269
279
  logger.warn('[embedder] No WebGPU adapter available — using CPU.');
package/src/utils/log-utils.ts CHANGED
@@ -84,41 +84,73 @@ const MAX_LOG_MESSAGE_LENGTH = 10_000;
84
84
  // separately for the console). Matching the ESC control byte is the point here.
85
85
  // eslint-disable-next-line no-control-regex
86
86
  const ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
87
- // Credentials embedded in a URL userinfo component: scheme://user:pass@host
88
- const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
87
+ // Credentials embedded in a URL userinfo component: scheme://user:pass@host.
88
+ //
89
+ // The scheme is length-bounded, and that bound is load-bearing rather than
90
+ // cosmetic. With an unbounded `[a-z0-9+.-]*`, every position inside a long
91
+ // alphanumeric run greedily consumes the rest of the input and then backtracks
92
+ // one character at a time looking for `://` — quadratic in the input length. On
93
+ // the 40_000-character window redactSecrets scans that measured ~1.4 SECONDS of
94
+ // blocked event loop per call, on a function that runs for every single log
95
+ // message, over content that is largely untrusted (scraped pages, provider error
96
+ // bodies): a page carrying one long unbroken token was enough to trigger it. A
97
+ // bounded scheme makes each start position O(32) instead of O(n) — same matches
98
+ // (a longer run simply starts matching nearer the `://`), ~215x faster measured.
99
+ // RFC 3986 schemes are short; no registered scheme approaches 32 characters.
100
+ const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]{0,31}:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
101
+ // Left/right boundaries for secret key names and opaque token formats. `\b`
102
+ // is WRONG here: `_` is a word character, so `\b` finds no boundary between the
103
+ // prefix and the secret in the single most common credential shape there is —
104
+ // an env var like PI_RESEARCH_API_KEY=…, OPENAI_API_KEY=…, DB_PASSWORD=…. Every
105
+ // such line was written to the log in the clear while the unprefixed `api_key=`
106
+ // was masked. Treat `_` (and any other non-alphanumeric) as a boundary, so a
107
+ // prefixed name matches while a name merely CONTAINING the word does not
108
+ // ("monkey=1" must not read as "key=1").
109
+ const B_LEFT = '(?<![A-Za-z0-9])';
110
+ const B_RIGHT = '(?![A-Za-z0-9])';
89
111
  // Cookie / Set-Cookie headers. The KV value class below stops at whitespace, so
90
112
  // "Cookie: lang=en; SID=secret123" would mask only the first pair and leave every
91
113
  // later one exposed. A cookie string is a ;-and-space separated list of secrets,
92
114
  // not a single token — mask the ENTIRE header value to end-of-line.
93
- const COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
115
+ const COOKIE_HEADER_PATTERN = new RegExp(
116
+ `${B_LEFT}(set[_-]?cookie|cookie)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\r\\n]+)`,
117
+ 'gi',
118
+ );
94
119
  // key=value / key: "value" pairs whose key names a secret. Includes bare
95
120
  // `key` (not just `api_key`/`private_key`): several provider query-param
96
121
  // APIs (e.g. StackExchange's `?key=<apikey>`) name the credential parameter
97
122
  // with no qualifying prefix. Over-redacting an unrelated "key" field (a
98
123
  // sort key, a map key) is the accepted trade-off — same one already made for
99
124
  // the equally generic bare `session`/`cookie` entries below.
100
- const SENSITIVE_KV_PATTERN =
101
- /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
125
+ const SENSITIVE_KV_PATTERN = new RegExp(
126
+ `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}` +
127
+ `(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
128
+ 'gi',
129
+ );
102
130
  // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
103
- const KNOWN_TOKEN_PATTERN =
104
- /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
131
+ const KNOWN_TOKEN_PATTERN = new RegExp(
132
+ `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
133
+ 'g',
134
+ );
105
135
  // JSON Web Tokens — header.payload.signature, each a base64url segment.
106
- const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
136
+ const JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, 'g');
107
137
  // HTTP Basic credentials: "Basic <base64>".
108
- const BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
138
+ const BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, 'gi');
109
139
  // HTTP Bearer credentials: "Bearer <token>". The token is often opaque — many
110
140
  // providers (e.g. GLM/zhipuai) use dotted keys that are NOT JWTs and carry no
111
141
  // known prefix, so KNOWN_TOKEN_PATTERN/JWT_PATTERN miss them and the KV pass
112
142
  // only masks the word "Bearer" (its value class stops at the space). Match the
113
143
  // scheme and redact the whole RFC 6750 b64token that follows, whatever its shape.
114
- const BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
144
+ const BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, 'gi');
115
145
  // Provider keys not covered by KNOWN_TOKEN_PATTERN: Google API/OAuth, Stripe.
116
- const PROVIDER_TOKEN_PATTERN =
117
- /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
146
+ const PROVIDER_TOKEN_PATTERN = new RegExp(
147
+ `${B_LEFT}(AIza[0-9A-Za-z_-]{35}|ya29\\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})${B_RIGHT}`,
148
+ 'g',
149
+ );
118
150
  // Long opaque hex secrets (>=32 hex chars), e.g. the 64-hex browser auth secret
119
151
  // or MD5/SHA-style tokens. Diagnostic logs may over-redact content hashes of
120
152
  // this length; that trade-off favors not persisting credentials clear-text.
121
- const LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
153
+ const LONG_HEX_SECRET_PATTERN = new RegExp(`${B_LEFT}[0-9a-fA-F]{32,}${B_RIGHT}`, 'g');
122
154
  // All C0 control characters plus DEL — newlines/carriage returns are the log/
123
155
  // terminal-injection vector when written to a raw console line. Matching these
124
156
  // control bytes is exactly the intent.
package/src/utils/user-agent.ts CHANGED
@@ -1,28 +1,39 @@
1
1
  /**
2
2
  * Realistic User-Agent Strings for Browser Rotation
3
3
  *
4
- * Provides current, realistic User-Agent strings that mimic real browsers.
5
- * Updated regularly to match current browser versions.
4
+ * Sent by the FETCH scrape path (web-scraper.ts); the stealth browser path
5
+ * supplies its own fingerprint-consistent UA via camoufox.
6
+ *
7
+ * These have a shelf life. Both browsers ship a major version roughly every
8
+ * four weeks, and a UA more than a few majors behind stable is itself a
9
+ * bot signal — the exact thing rotating them is meant to avoid. Refresh the
10
+ * two Chrome and two Firefox majors below when they drift; keep the platform
11
+ * tokens as-is (Chrome freezes the macOS token at 10_15_7 and zeroes the
12
+ * minor version triplet, Firefox mirrors its major in both `rv:` and the
13
+ * trailing `Firefox/` token).
14
+ *
15
+ * Last refreshed 2026-08-14 against Chrome 151 stable (Chrome Releases,
16
+ * 2026-07-28) and Firefox 153 stable (firefox.com release notes, 2026-08-11).
6
17
  */
7
18
 
8
19
  export const USER_AGENTS = [
9
- // Chrome on Windows (updated 2025)
10
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
11
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
20
+ // Chrome on Windows
21
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
22
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
12
23
 
13
24
  // Chrome on macOS
14
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
15
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
25
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
26
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
16
27
 
17
28
  // Chrome on Linux
18
- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
19
- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
29
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
30
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
20
31
 
21
- // Firefox on Windows (updated 2025)
22
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0',
23
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0',
32
+ // Firefox on Windows
33
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0',
34
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0',
24
35
 
25
36
  // Firefox on macOS
26
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0',
27
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:138.0) Gecko/20100101 Firefox/138.0',
28
- ];
37
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0',
38
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0',
39
+ ];
package/src/web-research/scraper-utils.ts CHANGED
@@ -379,6 +379,18 @@ function isPrivateIpv6(ip: string): boolean {
379
379
  // metadata service) — block the whole prefix regardless of what it wraps,
380
380
  // same rationale as the 6to4/Teredo ranges just above.
381
381
  if (normalized.startsWith('64:ff9b:')) return true;
382
+ // IPv4-COMPATIBLE ::/96 (RFC 4291 §2.5.5.1, deprecated) — the sibling of the
383
+ // ::ffff:0:0/96 mapped range decoded above, and the last IPv4-in-IPv6 form
384
+ // left open. `http://[::127.0.0.1]/` normalizes to `[::7f00:1]`, which matches
385
+ // none of the checks above: not `::1`, no `fc`/`fd`/`fe8x` prefix, no `::ffff:`
386
+ // prefix for extractMappedIpv4 to decode, and not one of the tunnel prefixes.
387
+ // Whether it reaches the embedded IPv4 depends on the OS, an intervening
388
+ // proxy, and NAT64 translation — the same "depends on the path" property that
389
+ // made 6to4/Teredo/NAT64 whole-range blocks the right call, so block the whole
390
+ // /96 rather than decode it. Deprecated with no legitimate scrape target.
391
+ // Matched structurally (top 96 bits zero => at most two groups after `::`),
392
+ // which is why it cannot swallow `::ffff:7f00:1` (three groups, ffff set).
393
+ if (/^::(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f]{1,4}(?::[0-9a-f]{1,4})?)$/.test(normalized)) return true;
382
394
  // All-zeros (unspecified address)
383
395
  if (normalized === '::' || normalized === '0:0:0:0:0:0:0:0') return true;
384
396
  return false;
package/src/web-research/web-scraper.ts CHANGED
@@ -250,13 +250,18 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
250
250
  const MAX_REDIRECTS = 10;
251
251
  let currentUrl = url;
252
252
  let response!: Response;
253
+ // One UA for the whole redirect chain. Picked per hop, a request could
254
+ // present as Chrome on Windows and then as Firefox on macOS while following
255
+ // a single Location chain — a browser never does that, so the rotation meant
256
+ // to blend in was instead handing out a distinctive signal.
257
+ const userAgent = getRandomUserAgent();
253
258
 
254
259
  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
255
260
  response = await fetch(currentUrl, {
256
261
  signal: controller.signal,
257
262
  redirect: 'manual',
258
263
  headers: {
259
- 'User-Agent': getRandomUserAgent(),
264
+ 'User-Agent': userAgent,
260
265
  'Accept': 'text/html,application/xhtml+xml,application/pdf,*/*;q=0.8',
261
266
  },
262
267
  // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.