@lincoln504/pi-research 1.3.8 → 1.3.10

package/src/utils/log-utils.ts

@@ -84,41 +84,73 @@ const MAX_LOG_MESSAGE_LENGTH = 10_000;
 // separately for the console). Matching the ESC control byte is the point here.
 // eslint-disable-next-line no-control-regex
 const ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
-// Credentials embedded in a URL userinfo component: scheme://user:pass@host
-const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
+// Credentials embedded in a URL userinfo component: scheme://user:pass@host.
+//
+// The scheme is length-bounded, and that bound is load-bearing rather than
+// cosmetic. With an unbounded `[a-z0-9+.-]*`, every position inside a long
+// alphanumeric run greedily consumes the rest of the input and then backtracks
+// one character at a time looking for `://` — quadratic in the input length. On
+// the 40_000-character window redactSecrets scans that measured ~1.4 SECONDS of
+// blocked event loop per call, on a function that runs for every single log
+// message, over content that is largely untrusted (scraped pages, provider error
+// bodies): a page carrying one long unbroken token was enough to trigger it. A
+// bounded scheme makes each start position O(32) instead of O(n) — same matches
+// (a longer run simply starts matching nearer the `://`), ~215x faster measured.
+// RFC 3986 schemes are short; no registered scheme approaches 32 characters.
+const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]{0,31}:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
+// Left/right boundaries for secret key names and opaque token formats. `\b`
+// is WRONG here: `_` is a word character, so `\b` finds no boundary between the
+// prefix and the secret in the single most common credential shape there is —
+// an env var like PI_RESEARCH_API_KEY=…, OPENAI_API_KEY=…, DB_PASSWORD=…. Every
+// such line was written to the log in the clear while the unprefixed `api_key=`
+// was masked. Treat `_` (and any other non-alphanumeric) as a boundary, so a
+// prefixed name matches while a name merely CONTAINING the word does not
+// ("monkey=1" must not read as "key=1").
+const B_LEFT = '(?<![A-Za-z0-9])';
+const B_RIGHT = '(?![A-Za-z0-9])';
 // Cookie / Set-Cookie headers. The KV value class below stops at whitespace, so
 // "Cookie: lang=en; SID=secret123" would mask only the first pair and leave every
 // later one exposed. A cookie string is a ;-and-space separated list of secrets,
 // not a single token — mask the ENTIRE header value to end-of-line.
-const COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
+const COOKIE_HEADER_PATTERN = new RegExp(
+  `${B_LEFT}(set[_-]?cookie|cookie)${B_RIGHT}(["']?\\s*[:=]\\s*["']?)([^\\r\\n]+)`,
+  'gi',
+);
 // key=value / key: "value" pairs whose key names a secret. Includes bare
 // `key` (not just `api_key`/`private_key`): several provider query-param
 // APIs (e.g. StackExchange's `?key=<apikey>`) name the credential parameter
 // with no qualifying prefix. Over-redacting an unrelated "key" field (a
 // sort key, a map key) is the accepted trade-off — same one already made for
 // the equally generic bare `session`/`cookie` entries below.
-const SENSITIVE_KV_PATTERN =
-  /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
+const SENSITIVE_KV_PATTERN = new RegExp(
+  `${B_LEFT}(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)${B_RIGHT}` +
+    `(["']?\\s*[:=]\\s*["']?)([^\\s"',&)]+)`,
+  'gi',
+);
 // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
-const KNOWN_TOKEN_PATTERN =
-  /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
+const KNOWN_TOKEN_PATTERN = new RegExp(
+  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
+  'g',
+);
 // JSON Web Tokens — header.payload.signature, each a base64url segment.
-const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
+const JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, 'g');
 // HTTP Basic credentials: "Basic <base64>".
-const BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
+const BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, 'gi');
 // HTTP Bearer credentials: "Bearer <token>". The token is often opaque — many
 // providers (e.g. GLM/zhipuai) use dotted keys that are NOT JWTs and carry no
 // known prefix, so KNOWN_TOKEN_PATTERN/JWT_PATTERN miss them and the KV pass
 // only masks the word "Bearer" (its value class stops at the space). Match the
 // scheme and redact the whole RFC 6750 b64token that follows, whatever its shape.
-const BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
+const BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, 'gi');
 // Provider keys not covered by KNOWN_TOKEN_PATTERN: Google API/OAuth, Stripe.
-const PROVIDER_TOKEN_PATTERN =
-  /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
+const PROVIDER_TOKEN_PATTERN = new RegExp(
+  `${B_LEFT}(AIza[0-9A-Za-z_-]{35}|ya29\\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})${B_RIGHT}`,
+  'g',
+);
 // Long opaque hex secrets (>=32 hex chars), e.g. the 64-hex browser auth secret
 // or MD5/SHA-style tokens. Diagnostic logs may over-redact content hashes of
 // this length; that trade-off favors not persisting credentials clear-text.
-const LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
+const LONG_HEX_SECRET_PATTERN = new RegExp(`${B_LEFT}[0-9a-fA-F]{32,}${B_RIGHT}`, 'g');
 // All C0 control characters plus DEL — newlines/carriage returns are the log/
 // terminal-injection vector when written to a raw console line. Matching these
 // control bytes is exactly the intent.

package/src/utils/user-agent.ts

@@ -1,28 +1,39 @@
 /**
  * Realistic User-Agent Strings for Browser Rotation
  *
- * Provides current, realistic User-Agent strings that mimic real browsers.
- * Updated regularly to match current browser versions.
+ * Sent by the FETCH scrape path (web-scraper.ts); the stealth browser path
+ * supplies its own fingerprint-consistent UA via camoufox.
+ *
+ * These have a shelf life. Both browsers ship a major version roughly every
+ * four weeks, and a UA more than a few majors behind stable is itself a
+ * bot signal — the exact thing rotating them is meant to avoid. Refresh the
+ * two Chrome and two Firefox majors below when they drift; keep the platform
+ * tokens as-is (Chrome freezes the macOS token at 10_15_7 and zeroes the
+ * minor version triplet, Firefox mirrors its major in both `rv:` and the
+ * trailing `Firefox/` token).
+ *
+ * Last refreshed 2026-08-14 against Chrome 151 stable (Chrome Releases,
+ * 2026-07-28) and Firefox 153 stable (firefox.com release notes, 2026-08-11).
  */
 
 export const USER_AGENTS = [
-    // Chrome on Windows (updated 2025)
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+    // Chrome on Windows
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
 
     // Chrome on macOS
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
 
     // Chrome on Linux
-    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',
 
-    // Firefox on Windows (updated 2025)
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0',
+    // Firefox on Windows
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0',
 
     // Firefox on macOS
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:138.0) Gecko/20100101 Firefox/138.0',
-];
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0',
+];

package/src/web-research/scraper-utils.ts

@@ -250,27 +250,73 @@ export function ssrfSafeLookup(
   });
 }
 
-// undici Agent used as the per-request dispatcher for scrape fetches. Cached
-// after first construction. `undefined` = not yet attempted, `null` = undici
-// unavailable (fall back to plain fetch with request-time validation only).
-let cachedSsrfDispatcher: unknown | null | undefined = undefined;
+/**
+ * True when the error is one of our SSRF blocks — request-time validation
+ * (message contains "not allowed") or the connect-time DNS pin (`ESSRFBLOCKED`).
+ *
+ * The connect-time error never surfaces directly: undici wraps it in
+ * `TypeError: fetch failed` with the real error down the `cause` chain, so a
+ * message check on the outer error alone files a connect-time block as a generic
+ * fetch error. Walk the chain (bounded — causes can be cyclic) instead.
+ */
+export function isSsrfBlockError(error: unknown): boolean {
+  let e: unknown = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if ((e as NodeJS.ErrnoException).code === 'ESSRFBLOCKED' || e.message.includes('not allowed')) {
+      return true;
+    }
+    e = (e as { cause?: unknown }).cause;
+  }
+  return false;
+}
+
+/** A fetch-shaped callable plus the dispatcher it must be invoked with. */
+export interface SsrfSafeFetcher {
+  /** undici's own fetch — NOT the global one. See getSsrfSafeFetcher. */
+  fetch: (url: string, init: Record<string, unknown>) => Promise<Response>;
+  dispatcher: unknown;
+}
+
+// Cached after first construction. `undefined` = not yet attempted, `null` =
+// undici unavailable (fall back to the global fetch with request-time
+// validation only).
+let cachedSsrfFetcher: SsrfSafeFetcher | null | undefined = undefined;
 
 /**
  * Build (once) an undici Agent whose connector resolves via ssrfSafeLookup, so
  * every scrape fetch — including each manually-followed redirect hop — connects
- * only to a validated public address. Returns null if undici can't be loaded;
- * callers then fall back to plain fetch (request-time validation still applies).
+ * only to a validated public address, PAIRED with the fetch implementation that
+ * can actually accept it.
+ *
+ * The pairing is the whole point. A dispatcher is only honoured by the undici
+ * that created it: Node's global `fetch` is backed by the undici BUILT INTO the
+ * runtime (6.21.0 on Node 22, 7.24.4 on Node 25), while this package depends on
+ * undici 8.x. Handing a v8 `Agent` to the global fetch throws
+ * `UND_ERR_INVALID_ARG: invalid onRequestStart method` on every request, on
+ * every supported Node — which is exactly what it did: the entire fetch scrape
+ * path failed instantly, every URL was retried once and then fell back to the
+ * stealth browser (~4.5s per page against ~250ms), and the connect-time SSRF pin
+ * this dispatcher exists for never engaged even once. The browser fallback made
+ * it invisible: scrapes still succeeded, just 15-20x slower.
+ *
+ * Returning the two together makes the invalid combination unrepresentable —
+ * a caller cannot end up holding a dispatcher without the fetch that honours it.
+ * Returns null if undici can't be loaded; callers then use the global fetch with
+ * no dispatcher (request-time validation still applies).
  */
-export async function getSsrfSafeDispatcher(): Promise<unknown | null> {
-  if (cachedSsrfDispatcher !== undefined) return cachedSsrfDispatcher as unknown | null;
+export async function getSsrfSafeFetcher(): Promise<SsrfSafeFetcher | null> {
+  if (cachedSsrfFetcher !== undefined) return cachedSsrfFetcher;
   try {
-    const { Agent } = await import('undici');
-    cachedSsrfDispatcher = new Agent({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent, fetch: undiciFetch } = await import('undici');
+    cachedSsrfFetcher = {
+      fetch: undiciFetch as unknown as SsrfSafeFetcher['fetch'],
+      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } }),
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher as unknown | null;
+  return cachedSsrfFetcher;
 }
 
 /**
@@ -379,6 +425,18 @@ function isPrivateIpv6(ip: string): boolean {
   // metadata service) — block the whole prefix regardless of what it wraps,
   // same rationale as the 6to4/Teredo ranges just above.
   if (normalized.startsWith('64:ff9b:')) return true;
+  // IPv4-COMPATIBLE ::/96 (RFC 4291 §2.5.5.1, deprecated) — the sibling of the
+  // ::ffff:0:0/96 mapped range decoded above, and the last IPv4-in-IPv6 form
+  // left open. `http://[::127.0.0.1]/` normalizes to `[::7f00:1]`, which matches
+  // none of the checks above: not `::1`, no `fc`/`fd`/`fe8x` prefix, no `::ffff:`
+  // prefix for extractMappedIpv4 to decode, and not one of the tunnel prefixes.
+  // Whether it reaches the embedded IPv4 depends on the OS, an intervening
+  // proxy, and NAT64 translation — the same "depends on the path" property that
+  // made 6to4/Teredo/NAT64 whole-range blocks the right call, so block the whole
+  // /96 rather than decode it. Deprecated with no legitimate scrape target.
+  // Matched structurally (top 96 bits zero => at most two groups after `::`),
+  // which is why it cannot swallow `::ffff:7f00:1` (three groups, ffff set).
+  if (/^::(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f]{1,4}(?::[0-9a-f]{1,4})?)$/.test(normalized)) return true;
   // All-zeros (unspecified address)
   if (normalized === '::' || normalized === '0:0:0:0:0:0:0:0') return true;
   return false;

package/src/web-research/web-scraper.ts

@@ -23,7 +23,7 @@ import {
 import {
   FETCH_LAYER_TIMEOUT,
 } from './types.ts';
-import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, formatErrorWithCause, isBenignScrapeFailure, } from './scraper-utils.ts';
+import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeFetcher, formatErrorWithCause, isBenignScrapeFailure, isSsrfBlockError, } from './scraper-utils.ts';
 import { isTransientError, abortableDelay } from './retry-utils.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { readBodyCapped, BodyTooLargeError } from '../utils/http-body.ts';
@@ -208,7 +208,7 @@ function decodeHtmlBody(bytes: Uint8Array, contentTypeHeader: string): string {
  * counts are precisely what production incidents are diagnosed from.
  */
 function trackFetchLayerFailure(url: string, error: unknown): void {
-  const isSsrf = error instanceof Error && error.message.includes('not allowed');
+  const isSsrf = isSsrfBlockError(error);
   errorTracker.trackError(error instanceof Error ? error : String(error), {
     component: 'scrapers',
     operation: 'fetch',
@@ -226,7 +226,12 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
   // closing the DNS-rebinding TOCTOU gap between validateUrlForSSRF (request
   // time) and fetch's own re-resolution (connect time). Null when undici is
   // unavailable — request-time validation above still applies.
-  const dispatcher = await getSsrfSafeDispatcher();
+  //
+  // The fetch implementation comes WITH the dispatcher and must be used with it:
+  // a dispatcher is only honoured by the undici that built it, and the global
+  // fetch is backed by Node's bundled undici, not ours. See getSsrfSafeFetcher.
+  const fetcher = await getSsrfSafeFetcher();
+  const doFetch = fetcher?.fetch ?? ((u: string, init: Record<string, unknown>) => fetch(u, init as RequestInit));
 
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), FETCH_LAYER_TIMEOUT);
@@ -250,18 +255,23 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     const MAX_REDIRECTS = 10;
     let currentUrl = url;
     let response!: Response;
+    // One UA for the whole redirect chain. Picked per hop, a request could
+    // present as Chrome on Windows and then as Firefox on macOS while following
+    // a single Location chain — a browser never does that, so the rotation meant
+    // to blend in was instead handing out a distinctive signal.
+    const userAgent = getRandomUserAgent();
 
     for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-      response = await fetch(currentUrl, {
+      response = await doFetch(currentUrl, {
         signal: controller.signal,
         redirect: 'manual',
         headers: {
-          'User-Agent': getRandomUserAgent(),
+          'User-Agent': userAgent,
           'Accept': 'text/html,application/xhtml+xml,application/pdf,*/*;q=0.8',
         },
-        // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.
-        ...(dispatcher ? { dispatcher } : {}),
-      } as RequestInit);
+        // Only ever attached alongside the fetch that honours it (see above).
+        ...(fetcher ? { dispatcher: fetcher.dispatcher } : {}),
+      });
 
       if (response.status >= 300 && response.status < 400) {
         const location = response.headers.get('location');
@@ -375,7 +385,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     // call in a retry, and tracking per attempt would record one failure twice —
     // inflating exactly the error counts these numbers are read to diagnose.
     // scrapeSingle calls trackFetchLayerFailure once with the final error.
-    if (error instanceof Error && error.message.includes('not allowed')) {
+    if (isSsrfBlockError(error)) {
       metrics.increment('scrape_operations_total', 1, { layer: 'fetch', status: 'ssrf_blocked' });
       metrics.observe('scrape_latency_ms', Date.now() - fetchStart, { layer: 'fetch', status: 'ssrf_blocked' });
       throw error;