@lincoln504/pi-research 1.3.9 → 1.3.10

package/dist/cli.mjs

@@ -2662,6 +2662,7 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
     PI_RESEARCH_LLM_THINKING_LEVEL: config.LLM_THINKING_LEVEL,
     PI_RESEARCH_PLANNING_MAX_TOKENS: String(config.PLANNING_MAX_TOKENS),
     PI_RESEARCH_SYNTHESIS_MAX_TOKENS: String(config.SYNTHESIS_MAX_TOKENS),
+    PI_RESEARCH_RESEARCHER_MAX_TOKENS: String(config.RESEARCHER_MAX_TOKENS),
     PI_RESEARCH_MIGRATION_STRATEGY: config.MIGRATION_STRATEGY,
     PI_RESEARCH_CONSOLE_LOG: String(config.CONSOLE_LOG),
     ...config.RESEARCH_MODEL ? { PI_RESEARCH_MODEL: config.RESEARCH_MODEL } : {},
@@ -2826,6 +2827,7 @@ function createConfig(env, processEnv) {
     LLM_THINKING_LEVEL: parseEnvEnum(e, "PI_RESEARCH_LLM_THINKING_LEVEL", ["off", "minimal", "low", "medium", "high"], DEFAULTS.LLM_THINKING_LEVEL),
     PLANNING_MAX_TOKENS: parseEnvNumber(e, "PI_RESEARCH_PLANNING_MAX_TOKENS", DEFAULTS.PLANNING_MAX_TOKENS, 1024, 131072),
     SYNTHESIS_MAX_TOKENS: parseEnvNumber(e, "PI_RESEARCH_SYNTHESIS_MAX_TOKENS", DEFAULTS.SYNTHESIS_MAX_TOKENS, 1024, 131072),
+    RESEARCHER_MAX_TOKENS: parseEnvNumber(e, "PI_RESEARCH_RESEARCHER_MAX_TOKENS", DEFAULTS.RESEARCHER_MAX_TOKENS, 1024, 131072),
     MIGRATION_STRATEGY: parseEnvEnum(e, "PI_RESEARCH_MIGRATION_STRATEGY", ["drop", "re-embed", "backup"], DEFAULTS.MIGRATION_STRATEGY),
     CONSOLE_LOG: parseEnvBool(e, "PI_RESEARCH_CONSOLE_LOG", DEFAULTS.CONSOLE_LOG),
     RESEARCH_MODEL: parseEnvString(e, "PI_RESEARCH_MODEL", DEFAULTS.RESEARCH_MODEL),
@@ -3013,6 +3015,15 @@ var init_config = __esm({
        *  the model's real maxTokens at call time.
        *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
       SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
+      /** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
+       *  Without a cap the session inherits the model's full catalog ceiling as the
+       *  per-request max_tokens — ~230k for large-context models — which providers that
+       *  budget-check max_tokens up front (OpenRouter 402s against remaining credits)
+       *  reject outright on low balances, even though the actual response would cost
+       *  cents. Default 16384: researcher turns are tool calls plus a cited report, far
+       *  below this. Clamped to the model's real ceiling at session build time.
+       *  Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
+      RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
       /** LLM Model override for researcher sub-agents and knowledge synthesis.
        *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
        *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
@@ -3090,6 +3101,7 @@ var init_config = __esm({
       "PI_RESEARCH_LLM_THINKING_LEVEL",
       "PI_RESEARCH_PLANNING_MAX_TOKENS",
       "PI_RESEARCH_SYNTHESIS_MAX_TOKENS",
+      "PI_RESEARCH_RESEARCHER_MAX_TOKENS",
       "PI_RESEARCH_MIGRATION_STRATEGY",
       "PI_RESEARCH_CONSOLE_LOG",
       "PI_RESEARCH_MODEL",
@@ -3132,8 +3144,7 @@ function getMaxScrapeBatches(config) {
   }
 }
 function resolveExcludedTools(callerExclusions, configDisabled = []) {
-  const base = callerExclusions && callerExclusions.length > 0 ? callerExclusions : DEFAULT_EXCLUDED_TOOLS;
-  return [.../* @__PURE__ */ new Set([...base, ...configDisabled])];
+  return [.../* @__PURE__ */ new Set([...DEFAULT_EXCLUDED_TOOLS, ...callerExclusions ?? [], ...configDisabled])];
 }
 var MAX_GATHERING_CALLS, MAX_GREP_CALLS, MAX_SCRAPE_URLS, MAX_TEAM_SIZE_LEVEL_1, MAX_TEAM_SIZE_LEVEL_2, MAX_TEAM_SIZE_LEVEL_3, MAX_ROUNDS_LEVEL_1, MAX_ROUNDS_LEVEL_2, MAX_ROUNDS_LEVEL_3, MAX_EXTRA_ROUNDS_WITH_STEERING, OSV_TIMEOUT_MS, WEBGPU_PROBE_TIMEOUT_MS, MAX_QUERY_LENGTH, MAX_FILENAME_QUERY_LENGTH, MIN_QUERY_LENGTH, MAX_EXPORT_RETRIES, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS, DEFAULT_MODEL_CONTEXT_WINDOW, BATCH_2_DEFAULT_CONCURRENCY, MAX_SCRAPE_CONTENT_CHARS_PER_DOC, RESEARCHER_LAUNCH_DELAY_MS, MAX_QUERIES_PER_RESEARCHER_LEVEL_1, MAX_QUERIES_PER_RESEARCHER_LEVEL_2, MAX_QUERIES_PER_RESEARCHER_LEVEL_3, DEFAULT_EXCLUDED_TOOLS;
 var init_constants = __esm({
@@ -17024,17 +17035,30 @@ function ssrfSafeLookup(hostname, options, callback, resolveImpl = dnsCb.lookup)
     return callback(null, safe[0].address, safe[0].family);
   });
 }
-var cachedSsrfDispatcher = void 0;
-async function getSsrfSafeDispatcher() {
-  if (cachedSsrfDispatcher !== void 0) return cachedSsrfDispatcher;
+function isSsrfBlockError(error) {
+  let e = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if (e.code === "ESSRFBLOCKED" || e.message.includes("not allowed")) {
+      return true;
+    }
+    e = e.cause;
+  }
+  return false;
+}
+var cachedSsrfFetcher = void 0;
+async function getSsrfSafeFetcher() {
+  if (cachedSsrfFetcher !== void 0) return cachedSsrfFetcher;
   try {
-    const { Agent: Agent2 } = await import("undici");
-    cachedSsrfDispatcher = new Agent2({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent: Agent2, fetch: undiciFetch } = await import("undici");
+    cachedSsrfFetcher = {
+      fetch: undiciFetch,
+      dispatcher: new Agent2({ connect: { lookup: ssrfSafeLookup } })
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher;
+  return cachedSsrfFetcher;
 }
 function formatErrorWithCause(err) {
   try {
@@ -17284,7 +17308,7 @@ function decodeHtmlBody(bytes, contentTypeHeader) {
   }
 }
 function trackFetchLayerFailure(url, error) {
-  const isSsrf = error instanceof Error && error.message.includes("not allowed");
+  const isSsrf = isSsrfBlockError(error);
   errorTracker.trackError(error instanceof Error ? error : String(error), {
     component: "scrapers",
     operation: "fetch",
@@ -17296,7 +17320,8 @@ function trackFetchLayerFailure(url, error) {
 }
 async function scrapeWithFetch(url, signal) {
   await validateUrlForSSRF(url);
-  const dispatcher = await getSsrfSafeDispatcher();
+  const fetcher = await getSsrfSafeFetcher();
+  const doFetch = fetcher?.fetch ?? ((u, init) => fetch(u, init));
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), FETCH_LAYER_TIMEOUT);
   safeUnref(timeoutId);
@@ -17315,15 +17340,15 @@ async function scrapeWithFetch(url, signal) {
     let response;
     const userAgent = getRandomUserAgent();
     for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-      response = await fetch(currentUrl, {
+      response = await doFetch(currentUrl, {
         signal: controller.signal,
         redirect: "manual",
         headers: {
           "User-Agent": userAgent,
           "Accept": "text/html,application/xhtml+xml,application/pdf,*/*;q=0.8"
         },
-        // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.
-        ...dispatcher ? { dispatcher } : {}
+        // Only ever attached alongside the fetch that honours it (see above).
+        ...fetcher ? { dispatcher: fetcher.dispatcher } : {}
       });
       if (response.status >= 300 && response.status < 400) {
         const location = response.headers.get("location");
@@ -17424,7 +17449,7 @@ async function scrapeWithFetch(url, signal) {
     metrics.observe("scrape_latency_ms", fetchDuration, { layer: "fetch", content_type: "html", status: "success" });
     return { source: "fetch", layer: "fetch", markdown };
   } catch (error) {
-    if (error instanceof Error && error.message.includes("not allowed")) {
+    if (isSsrfBlockError(error)) {
       metrics.increment("scrape_operations_total", 1, { layer: "fetch", status: "ssrf_blocked" });
       metrics.observe("scrape_latency_ms", Date.now() - fetchStart, { layer: "fetch", status: "ssrf_blocked" });
       throw error;
@@ -21524,6 +21549,8 @@ async function createResearcherSession(options) {
     cwd
   });
   logger.info(`[Researcher] Using model for researcher ${researcherId}: ${modelToUse.provider}/${modelToUse.id}`);
+  const outputCap = config?.RESEARCHER_MAX_TOKENS ?? 16384;
+  const sessionModel = modelToUse.maxTokens > outputCap ? { ...modelToUse, maxTokens: outputCap } : modelToUse;
   try {
     const tracker = new ToolUsageTracker(createDefaultToolLimits(config));
     const globalLinks = updateGlobalLinks || (() => {
@@ -21564,7 +21591,7 @@ async function createResearcherSession(options) {
       sessionManager: SessionManager.inMemory(),
       // Each researcher gets its own isolated session
       settingsManager: researcherSettings,
-      model: modelToUse,
+      model: sessionModel,
       // pi 0.80.8 replaced the SDK's `modelRegistry`/`authStorage` session options
       // with the async `modelRuntime`. Pass the runtime backing OUR registry when
       // this run built one (standalone CLI/SDK): it carries any explicit API key
@@ -21595,7 +21622,7 @@ async function createResearcherSession(options) {
         throw new Error("Session creation returned invalid result");
       }
       sessionRef.session = result.session;
-      return { session: result.session, resolvedModel: modelToUse };
+      return { session: result.session, resolvedModel: sessionModel };
     } catch (tailError) {
       const session = result?.session;
       if (session && typeof session.abort === "function") {
@@ -21618,7 +21645,7 @@ async function createResearcherSession(options) {
 
 // src/orchestration/researcher-executor.ts
 init_text_utils();
-init_llm();
+init_llm_usage();
 init_logger();
 init_metrics();
 init_service_interfaces();
@@ -22054,12 +22081,13 @@ ${prompt}`);
         if (msg2?.["role"] !== "assistant") return;
         const rawUsage = msg2["usage"];
         if (rawUsage) {
-          const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
+          const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+            component: "researcher",
+            complexity,
+            observer: { onTokensConsumed: (t, c) => safeObserve(() => observer?.onTokensConsumed?.(t, c)) }
+          });
           if (tokens > 0 || cost > 0) {
-            metrics.increment("llm_tokens_total", tokens, { component: "researcher", complexity: String(complexity) });
-            metrics.increment("llm_cost_total", cost, { component: "researcher", complexity: String(complexity) });
             safeObserve(() => observer?.onResearcherProgress?.(id, void 0, tokens, cost));
-            safeObserve(() => observer?.onTokensConsumed?.(tokens, cost));
           }
         }
       } else if (event.type === "tool_execution_start") {
@@ -22186,7 +22214,7 @@ ${responseText}`);
 
 // src/orchestration/quick-research-orchestrator.ts
 init_prompts();
-init_llm();
+init_llm_usage();
 init_logger();
 init_config();
 init_text_utils();
@@ -22467,12 +22495,13 @@ ${prompt}`);
           if (msg2?.["role"] !== "assistant") return;
           const rawUsage = msg2["usage"];
           if (rawUsage) {
-            const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
+            const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+              component: "quick_researcher",
+              complexity: 0,
+              observer
+            });
             if (tokens > 0 || cost > 0) {
-              metrics.increment("llm_tokens_total", tokens, { component: "quick_researcher", complexity: "0" });
-              metrics.increment("llm_cost_total", cost, { component: "quick_researcher", complexity: "0" });
               observer?.onResearcherProgress?.("quick", void 0, tokens, cost);
-              observer?.onTokensConsumed?.(tokens, cost);
             }
           }
         } else if (event.type === "tool_execution_start") {

package/dist/thread-worker.mjs

@@ -1512,8 +1512,9 @@ __export(scraper_utils_exports, {
   extractDomain: () => extractDomain,
   formatErrorWithCause: () => formatErrorWithCause,
   getRandomUserAgent: () => getRandomUserAgent,
-  getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
+  getSsrfSafeFetcher: () => getSsrfSafeFetcher,
   isBenignScrapeFailure: () => isBenignScrapeFailure,
+  isSsrfBlockError: () => isSsrfBlockError,
   ssrfSafeLookup: () => ssrfSafeLookup,
   stripImageLinks: () => stripImageLinks,
   validateContent: () => validateContent,
@@ -1633,16 +1634,29 @@ function ssrfSafeLookup(hostname, options, callback, resolveImpl = dnsCb.lookup)
     return callback(null, safe[0].address, safe[0].family);
   });
 }
-async function getSsrfSafeDispatcher() {
-  if (cachedSsrfDispatcher !== void 0) return cachedSsrfDispatcher;
+function isSsrfBlockError(error) {
+  let e = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if (e.code === "ESSRFBLOCKED" || e.message.includes("not allowed")) {
+      return true;
+    }
+    e = e.cause;
+  }
+  return false;
+}
+async function getSsrfSafeFetcher() {
+  if (cachedSsrfFetcher !== void 0) return cachedSsrfFetcher;
   try {
-    const { Agent } = await import("undici");
-    cachedSsrfDispatcher = new Agent({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent, fetch: undiciFetch } = await import("undici");
+    cachedSsrfFetcher = {
+      fetch: undiciFetch,
+      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher;
+  return cachedSsrfFetcher;
 }
 function formatErrorWithCause(err) {
   try {
@@ -1776,7 +1790,7 @@ function validateContent(html, markdown, url) {
     throw error;
   }
 }
-var cachedSsrfDispatcher;
+var cachedSsrfFetcher;
 var init_scraper_utils = __esm({
   "src/web-research/scraper-utils.ts"() {
     "use strict";
@@ -1785,7 +1799,7 @@ var init_scraper_utils = __esm({
     init_logger();
     init_scraper_types();
     init_browser_error_utils();
-    cachedSsrfDispatcher = void 0;
+    cachedSsrfFetcher = void 0;
   }
 });
 
@@ -2530,6 +2544,15 @@ var ConfigSchema = Type.Object({
    *  the model's real maxTokens at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
   SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
+  /** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
+   *  Without a cap the session inherits the model's full catalog ceiling as the
+   *  per-request max_tokens — ~230k for large-context models — which providers that
+   *  budget-check max_tokens up front (OpenRouter 402s against remaining credits)
+   *  reject outright on low balances, even though the actual response would cost
+   *  cents. Default 16384: researcher turns are tool calls plus a cited report, far
+   *  below this. Clamped to the model's real ceiling at session build time.
+   *  Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
+  RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
   /** LLM Model override for researcher sub-agents and knowledge synthesis.
    *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
    *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)

package/docs/ARCHITECTURE.md

@@ -115,12 +115,14 @@ agent to invoke:
 In deep research `search` is excluded — the coordinator runs the search burst and hands
 out URLs directly.
 
-`grep` is excluded by default at **every** depth and on every front-end (CLI, SDK, agent
-skill, pi extension): this is web research, and a capable model otherwise spends turns
-searching the local filesystem. Passing an explicit `excludeTools` list — `--exclude-tools`
-on the CLI, the `excludeTools` tool parameter in the extension — REPLACES that default, so
-naming other tools re-enables `grep`. `PI_RESEARCH_DISABLED_TOOLS` is separate and strictly
-additive: it can only ever remove capabilities (see [CONFIGURATION.md](CONFIGURATION.md)).
+`grep` is excluded at **every** depth and on every front-end (CLI, SDK, agent skill,
+pi extension): this is web research, and a capable model otherwise spends turns
+searching the local filesystem. Both exclusion surfaces — the `excludeTools` list
+(`--exclude-tools` on the CLI, the `excludeTools` tool parameter in the extension) and
+`PI_RESEARCH_DISABLED_TOOLS` — are strictly additive on top of that default: they can
+only ever remove capabilities (see [CONFIGURATION.md](CONFIGURATION.md)). Before 1.3.10
+a non-empty `excludeTools` list replaced the default, so naming any other tool silently
+re-enabled `grep`.
 
 Researchers cannot write files, run shell commands, or reach the network outside these
 tools.

package/docs/CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.10] - 2026-08-14
+
+The post-release verification round for 1.3.9 kept digging and found that the
+fetch scrape path — the fast primary transport in front of the stealth browser —
+had been dead in production on every supported Node version, hidden completely by
+the browser fallback. Three more researcher-session defects surfaced from the same
+investigation, two of them demonstrated live against a real provider. Each fix
+carries a regression test verified to fail against the pre-fix code (the
+accounting fix pins the deep-researcher site; the quick site shares the same
+helper call), and the fetch fix carries the first test of that layer that uses a
+real socket instead of a mocked `fetch`. An independent fresh-eyes review of the
+whole delta then caught two follow-on defects, fixed below.
+
+### Fixed
+
+- **Every fetch scrape failed instantly; the whole path silently ran on the browser fallback.** `getSsrfSafeDispatcher()` built an `Agent` from the undici 8.x this package depends on and handed it to Node's **global** `fetch()` — which is backed by the undici compiled *into* the runtime (6.x on Node 22, 7.x on Node 25). Cross-major dispatchers are rejected outright (`UND_ERR_INVALID_ARG: invalid onRequestStart method`), so every fetch attempt threw before a single packet left the machine, burned one retry, and fell back to the stealth browser: ~3.4 s per page against the ~0.9 s the fetch path delivers, the browser pool saturated with work it never needed — and the connect-time SSRF DNS pin, the entire reason the dispatcher exists, never engaged once (request-time validation still applied throughout, so this was defence-in-depth loss, not an open hole). Nothing caught it because every test of the layer stubs the global `fetch`, and the fallback kept scrapes succeeding. The replacement, `getSsrfSafeFetcher()`, returns the Agent *paired with undici's own `fetch`*, making the invalid combination unrepresentable, and a new test drives the real fetch path against a real loopback socket with the browser layer disabled — it fails against the pre-fix code with the exact production error. Measured on live runs: 0 fetch / 5 browser successes before, 4 fetch / 0 browser after.
+- **Researchers sent the model's full ~230k catalog ceiling as `max_tokens` — and OpenRouter 402'd every call on a low balance.** The researcher session inherited the model's catalog `maxTokens` as its per-request output cap. Providers that budget-check `max_tokens` up front rejected the call before running it: observed live as `402: You requested up to 231037 tokens, but can only afford 24467` on every researcher, failing entire runs whose real cost was cents. Researcher output per call — tool calls plus a cited report — is nowhere near that. New `PI_RESEARCH_RESEARCHER_MAX_TOKENS` knob (default 16384, range 1024–131072), clamped to the model's real ceiling, applied in the shared session factory so deep and quick sub-agents are both covered.
+- **`--exclude-tools` silently granted researchers local-filesystem `grep`.** A non-empty exclusion list *replaced* the default exclusions instead of adding to them, so `--exclude-tools stackexchange` enabled a capability as a side effect of a flag documented as "disable internal tools". The same defect was fixed for `PI_RESEARCH_DISABLED_TOOLS` in 1.3.0 with the rationale "a subtract-only setting must never grant a capability"; the caller-facing surface kept the replacement semantics until now. Both surfaces are strictly additive over the default exclusions. (Behavioral change: there is no longer any way to hand researchers `grep`, which ARCHITECTURE.md previously documented as a quirk of the replacement rule.)
+- **A misconfigured $0 price table was silent on depth-0 runs.** Both researcher `message_end` accounting sites hand-copied the extract→increment→emit block instead of calling `recordLlmUsage`, so `warnIfUnpriced` — added in 1.1.2 precisely because a hand-written `models.json` entry silently discards pi's pricing — never ran for researcher usage. Depth-1+ runs still warned via the coordinator; depth-0 runs have no coordinator and reported `$0.0000` with no diagnostic at all. Metric labels and observer payloads are unchanged (`onTokensConsumed` now fires before `onResearcherProgress` rather than after — no consumer is order-sensitive), and the deep path keeps its `safeObserve` isolation.
+- **A connect-time SSRF block was filed as a generic fetch error.** undici never surfaces the connector's `ESSRFBLOCKED` error directly — `fetch` throws `TypeError: fetch failed` with the real error down the `cause` chain — so the message-based classification at both tracking sites missed it and a DNS-rebinding block would have been invisible in the SSRF metrics. Unreachable before this release (the dispatcher path had never executed); now that it is live, `isSsrfBlockError()` walks the bounded cause chain. Diagnostics-only: the block itself always held.
+- **One unit test silently made live network requests.** The fetch-stub migration for the undici change covered two of the three test files that drive the scrape layer; the missed one (`scraper-pdf-native-module-failure`) kept stubbing the *global* fetch, so every run of that "hermetic" test performed real DNS and an outbound TLS connection to a third-party host — and passed only because that host's certificate mismatch happened to reproduce the stubbed failure. Found by the fresh-eyes review; it now shims `getSsrfSafeFetcher` like its siblings.
+
+### Verified
+
+- Full suites green after the changes: 191 unit files / 2457 tests, integration 53+8 skipped and 103, lint, both type-checks, dependency rules.
+- Live production runs on OpenRouter at depths 0, 1, and 2: reports with citations, fetch-path scrapes at ~0.6–1.5 s/page with zero browser fallbacks and zero undici errors, non-zero cost accounting, knowledge store round-trip (same-day scrapes retrieved and grounded).
+- The stderr FD-2 capture deliberately still skips pipes/sockets: when a parent process is reading stderr (pi host, MCP, CI), stealing FD 2 would hide fatal diagnostics to suppress cosmetic Dawn noise. Reviewed and kept as designed.
+
 ## [1.3.9] - 2026-08-14
 
 A whole-tree audit sweep. Six defects, each found by looking at what the code

package/docs/CONFIGURATION.md

@@ -114,6 +114,7 @@ These are advanced, env-only knobs (not in the TUI).
 | `PI_RESEARCH_LLM_THINKING_LEVEL` | `off` | off · minimal · low · medium · high | Chain-of-thought level for all engine LLM work (coordinator, evaluator, synthesis, JSON-repair, knowledge extraction, and researcher sub-agents). Off by default — these calls emit structured JSON / cited reports, so a thinking block only consumes the output budget and can truncate the answer. Clamped per model by pi. |
 | `PI_RESEARCH_PLANNING_MAX_TOKENS` | `16384` | 1024–131072 | Max output tokens for the plan + mid-round evaluator decision. Clamped to the model's real ceiling. |
 | `PI_RESEARCH_SYNTHESIS_MAX_TOKENS` | `32768` | 1024–131072 | Max output tokens for the final synthesized report. Clamped to the model's real ceiling. |
+| `PI_RESEARCH_RESEARCHER_MAX_TOKENS` | `16384` | 1024–131072 | Max output tokens per researcher LLM call (deep and quick). Without it the model's full catalog ceiling is sent as `max_tokens`, which OpenRouter budget-checks against remaining credits and rejects on low balances. |
 
 Knowledge store
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.3.9",
+  "version": "1.3.10",
   "description": "Free unlimited web search and deep research for the Pi coding agent. Local knowledge store. Also works from Claude Code, Codex, and OpenClaw.",
   "type": "module",
   "private": false,

package/skills/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.3.9", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.3.10", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---
 

package/src/config.ts

@@ -122,6 +122,15 @@ export const ConfigSchema = Type.Object({
    *  the model's real maxTokens at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
   SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
+  /** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
+   *  Without a cap the session inherits the model's full catalog ceiling as the
+   *  per-request max_tokens — ~230k for large-context models — which providers that
+   *  budget-check max_tokens up front (OpenRouter 402s against remaining credits)
+   *  reject outright on low balances, even though the actual response would cost
+   *  cents. Default 16384: researcher turns are tool calls plus a cited report, far
+   *  below this. Clamped to the model's real ceiling at session build time.
+   *  Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
+  RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
   /** LLM Model override for researcher sub-agents and knowledge synthesis.
    *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
    *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
@@ -233,6 +242,7 @@ const USER_MIGRATION_KEYS = [
   'PI_RESEARCH_LLM_THINKING_LEVEL',
   'PI_RESEARCH_PLANNING_MAX_TOKENS',
   'PI_RESEARCH_SYNTHESIS_MAX_TOKENS',
+  'PI_RESEARCH_RESEARCHER_MAX_TOKENS',
   'PI_RESEARCH_MIGRATION_STRATEGY',
   'PI_RESEARCH_CONSOLE_LOG',
   'PI_RESEARCH_MODEL',
@@ -865,6 +875,7 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
     PI_RESEARCH_LLM_THINKING_LEVEL: config.LLM_THINKING_LEVEL,
     PI_RESEARCH_PLANNING_MAX_TOKENS: String(config.PLANNING_MAX_TOKENS),
     PI_RESEARCH_SYNTHESIS_MAX_TOKENS: String(config.SYNTHESIS_MAX_TOKENS),
+    PI_RESEARCH_RESEARCHER_MAX_TOKENS: String(config.RESEARCHER_MAX_TOKENS),
     PI_RESEARCH_MIGRATION_STRATEGY: config.MIGRATION_STRATEGY,
     PI_RESEARCH_CONSOLE_LOG: String(config.CONSOLE_LOG),
     ...(config.RESEARCH_MODEL ? { PI_RESEARCH_MODEL: config.RESEARCH_MODEL } : {}),
@@ -1095,6 +1106,7 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
     LLM_THINKING_LEVEL: parseEnvEnum(e, 'PI_RESEARCH_LLM_THINKING_LEVEL', ['off', 'minimal', 'low', 'medium', 'high'] as const, DEFAULTS.LLM_THINKING_LEVEL),
     PLANNING_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_PLANNING_MAX_TOKENS', DEFAULTS.PLANNING_MAX_TOKENS, 1024, 131072),
     SYNTHESIS_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_SYNTHESIS_MAX_TOKENS', DEFAULTS.SYNTHESIS_MAX_TOKENS, 1024, 131072),
+    RESEARCHER_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_RESEARCHER_MAX_TOKENS', DEFAULTS.RESEARCHER_MAX_TOKENS, 1024, 131072),
     MIGRATION_STRATEGY: parseEnvEnum(e, 'PI_RESEARCH_MIGRATION_STRATEGY', ['drop', 're-embed', 'backup'] as const, DEFAULTS.MIGRATION_STRATEGY),
     CONSOLE_LOG: parseEnvBool(e, 'PI_RESEARCH_CONSOLE_LOG', DEFAULTS.CONSOLE_LOG),
     RESEARCH_MODEL: parseEnvString(e, 'PI_RESEARCH_MODEL', DEFAULTS.RESEARCH_MODEL),

package/src/constants.ts

@@ -185,22 +185,17 @@ export const DEFAULT_EXCLUDED_TOOLS: readonly string[] = ['grep'];
 /**
  * Resolve the effective researcher exclusion list.
  *
- * An EMPTY array is "no preference", not "exclude nothing". The two front-ends
- * express no-preference differently — the pi extension always builds an array
- * (`[...new Set([...])]`, often empty) while the CLI omits the key entirely — so a
- * plain truthiness test made the same user input mean opposite things: grep was
- * available in the extension and excluded on the CLI, regardless of depth.
- *
- * `configDisabled` (PI_RESEARCH_DISABLED_TOOLS) is strictly ADDITIVE. Folding it in
- * as a replacement meant disabling any unrelated tool silently ENABLED grep, so a
- * subtract-only setting could grant a capability.
+ * EVERY input is strictly ADDITIVE on top of DEFAULT_EXCLUDED_TOOLS. Both
+ * subtract-only surfaces — `--exclude-tools` / `excludeTools` and
+ * PI_RESEARCH_DISABLED_TOOLS (`configDisabled`) — are documented as "disable
+ * tools", so neither may GRANT a capability as a side effect. Each has held the
+ * replacement semantics at some point, with the same consequence both times:
+ * excluding any unrelated tool silently enabled local-filesystem grep for web
+ * researchers.
  */
 export function resolveExcludedTools(
   callerExclusions: readonly string[] | undefined,
   configDisabled: readonly string[] = [],
 ): string[] {
-  const base = callerExclusions && callerExclusions.length > 0
-    ? callerExclusions
-    : DEFAULT_EXCLUDED_TOOLS;
-  return [...new Set([...base, ...configDisabled])];
+  return [...new Set([...DEFAULT_EXCLUDED_TOOLS, ...(callerExclusions ?? []), ...configDisabled])];
 }

package/src/infrastructure/browser/thread-worker.mjs

@@ -1512,8 +1512,9 @@ __export(scraper_utils_exports, {
   extractDomain: () => extractDomain,
   formatErrorWithCause: () => formatErrorWithCause,
   getRandomUserAgent: () => getRandomUserAgent,
-  getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
+  getSsrfSafeFetcher: () => getSsrfSafeFetcher,
   isBenignScrapeFailure: () => isBenignScrapeFailure,
+  isSsrfBlockError: () => isSsrfBlockError,
   ssrfSafeLookup: () => ssrfSafeLookup,
   stripImageLinks: () => stripImageLinks,
   validateContent: () => validateContent,
@@ -1633,16 +1634,29 @@ function ssrfSafeLookup(hostname, options, callback, resolveImpl = dnsCb.lookup)
     return callback(null, safe[0].address, safe[0].family);
   });
 }
-async function getSsrfSafeDispatcher() {
-  if (cachedSsrfDispatcher !== void 0) return cachedSsrfDispatcher;
+function isSsrfBlockError(error) {
+  let e = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if (e.code === "ESSRFBLOCKED" || e.message.includes("not allowed")) {
+      return true;
+    }
+    e = e.cause;
+  }
+  return false;
+}
+async function getSsrfSafeFetcher() {
+  if (cachedSsrfFetcher !== void 0) return cachedSsrfFetcher;
   try {
-    const { Agent } = await import("undici");
-    cachedSsrfDispatcher = new Agent({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent, fetch: undiciFetch } = await import("undici");
+    cachedSsrfFetcher = {
+      fetch: undiciFetch,
+      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher;
+  return cachedSsrfFetcher;
 }
 function formatErrorWithCause(err) {
   try {
@@ -1776,7 +1790,7 @@ function validateContent(html, markdown, url) {
     throw error;
   }
 }
-var cachedSsrfDispatcher;
+var cachedSsrfFetcher;
 var init_scraper_utils = __esm({
   "src/web-research/scraper-utils.ts"() {
     "use strict";
@@ -1785,7 +1799,7 @@ var init_scraper_utils = __esm({
     init_logger();
     init_scraper_types();
     init_browser_error_utils();
-    cachedSsrfDispatcher = void 0;
+    cachedSsrfFetcher = void 0;
   }
 });
 
@@ -2530,6 +2544,15 @@ var ConfigSchema = Type.Object({
    *  the model's real maxTokens at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
   SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
+  /** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
+   *  Without a cap the session inherits the model's full catalog ceiling as the
+   *  per-request max_tokens — ~230k for large-context models — which providers that
+   *  budget-check max_tokens up front (OpenRouter 402s against remaining credits)
+   *  reject outright on low balances, even though the actual response would cost
+   *  cents. Default 16384: researcher turns are tool calls plus a cited report, far
+   *  below this. Clamped to the model's real ceiling at session build time.
+   *  Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
+  RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
   /** LLM Model override for researcher sub-agents and knowledge synthesis.
    *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
    *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)

package/src/orchestration/quick-research-orchestrator.ts

@@ -13,7 +13,7 @@ import {
 import { type Model } from '@earendil-works/pi-ai';
 import { injectCurrentDate } from '../core/llm/inject-date.ts';
 import { loadPrompt } from '../core/llm/prompts.ts';
-import { extractUsage } from '../types/llm.ts';
+import { recordLlmUsage } from '../utils/llm-usage.ts';
 import { logger } from '../logger.ts';
 import { getConfig, type Config } from '../config.ts';
 import { createResearcherSession } from './researcher.ts';
@@ -238,13 +238,13 @@ export class QuickResearchOrchestrator {
 
                 const rawUsage = msg['usage'] as any;
                 if (rawUsage) {
-                    const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
-
+                    const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+                        component: 'quick_researcher',
+                        complexity: 0,
+                        observer,
+                    });
                     if (tokens > 0 || cost > 0) {
-                        metrics.increment('llm_tokens_total', tokens, { component: 'quick_researcher', complexity: '0' });
-                        metrics.increment('llm_cost_total', cost, { component: 'quick_researcher', complexity: '0' });
                         observer?.onResearcherProgress?.('quick', undefined, tokens, cost);
-                        observer?.onTokensConsumed?.(tokens, cost);
                     }
                 }
             } else if (event.type === 'tool_execution_start') {

package/src/orchestration/researcher-executor.ts

@@ -9,7 +9,7 @@ import type { SystemResearchState } from './deep-research-types.ts';
 import { createResearcherSession } from './researcher.ts';
 import { registerScrapedLinks } from '../utils/shared-links.ts';
 import { ensureAssistantResponse } from '../utils/text-utils.ts';
-import { extractUsage } from '../types/llm.ts';
+import { recordLlmUsage } from '../utils/llm-usage.ts';
 import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
 import { ServiceNames,
@@ -347,13 +347,15 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
 
         const rawUsage = msg['usage'] as any;
         if (rawUsage) {
-          const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
-
+          // Observer exceptions must not kill the researcher, so hand recordLlmUsage
+          // a safeObserve-wrapped sink rather than the raw observer.
+          const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+            component: 'researcher',
+            complexity,
+            observer: { onTokensConsumed: (t, c) => safeObserve(() => observer?.onTokensConsumed?.(t, c)) },
+          });
           if (tokens > 0 || cost > 0) {
-            metrics.increment('llm_tokens_total', tokens, { component: 'researcher', complexity: String(complexity) });
-            metrics.increment('llm_cost_total', cost, { component: 'researcher', complexity: String(complexity) });
             safeObserve(() => observer?.onResearcherProgress?.(id, undefined, tokens, cost));
-            safeObserve(() => observer?.onTokensConsumed?.(tokens, cost));
           }
         }
       } else if (event.type === 'tool_execution_start') {

package/src/orchestration/researcher.ts

@@ -85,6 +85,17 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
 
   logger.info(`[Researcher] Using model for researcher ${researcherId}: ${modelToUse.provider}/${modelToUse.id}`);
 
+  // The session sends model.maxTokens as the per-request max_tokens. Large-context
+  // catalog entries carry ~230k ceilings, and providers that budget-check max_tokens
+  // up front (OpenRouter 402s it against remaining credits) then reject every
+  // researcher call on a low balance even though the real response costs cents.
+  // Clamp the session's copy of the model to RESEARCHER_MAX_TOKENS; Model is plain
+  // catalog data, so a spread clone is safe and the shared registry entry is untouched.
+  const outputCap = config?.RESEARCHER_MAX_TOKENS ?? 16384;
+  const sessionModel = modelToUse.maxTokens > outputCap
+    ? { ...modelToUse, maxTokens: outputCap }
+    : modelToUse;
+
   try {
     const tracker = new ToolUsageTracker(createDefaultToolLimits(config));
 
@@ -138,7 +149,7 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
       tools, // Explicit allowlist (BUG-1 fix)
       sessionManager: SessionManager.inMemory(), // Each researcher gets its own isolated session
       settingsManager: researcherSettings,
-      model: modelToUse,
+      model: sessionModel,
       // pi 0.80.8 replaced the SDK's `modelRegistry`/`authStorage` session options
       // with the async `modelRuntime`. Pass the runtime backing OUR registry when
       // this run built one (standalone CLI/SDK): it carries any explicit API key
@@ -185,7 +196,7 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
 
       sessionRef.session = result.session;
 
-      return { session: result.session, resolvedModel: modelToUse };
+      return { session: result.session, resolvedModel: sessionModel };
     } catch (tailError) {
       const session = result?.session;
       if (session && typeof session.abort === 'function') {

package/src/web-research/scraper-utils.ts

@@ -250,27 +250,73 @@ export function ssrfSafeLookup(
   });
 }
 
-// undici Agent used as the per-request dispatcher for scrape fetches. Cached
-// after first construction. `undefined` = not yet attempted, `null` = undici
-// unavailable (fall back to plain fetch with request-time validation only).
-let cachedSsrfDispatcher: unknown | null | undefined = undefined;
+/**
+ * True when the error is one of our SSRF blocks — request-time validation
+ * (message contains "not allowed") or the connect-time DNS pin (`ESSRFBLOCKED`).
+ *
+ * The connect-time error never surfaces directly: undici wraps it in
+ * `TypeError: fetch failed` with the real error down the `cause` chain, so a
+ * message check on the outer error alone files a connect-time block as a generic
+ * fetch error. Walk the chain (bounded — causes can be cyclic) instead.
+ */
+export function isSsrfBlockError(error: unknown): boolean {
+  let e: unknown = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if ((e as NodeJS.ErrnoException).code === 'ESSRFBLOCKED' || e.message.includes('not allowed')) {
+      return true;
+    }
+    e = (e as { cause?: unknown }).cause;
+  }
+  return false;
+}
+
+/** A fetch-shaped callable plus the dispatcher it must be invoked with. */
+export interface SsrfSafeFetcher {
+  /** undici's own fetch — NOT the global one. See getSsrfSafeFetcher. */
+  fetch: (url: string, init: Record<string, unknown>) => Promise<Response>;
+  dispatcher: unknown;
+}
+
+// Cached after first construction. `undefined` = not yet attempted, `null` =
+// undici unavailable (fall back to the global fetch with request-time
+// validation only).
+let cachedSsrfFetcher: SsrfSafeFetcher | null | undefined = undefined;
 
 /**
  * Build (once) an undici Agent whose connector resolves via ssrfSafeLookup, so
  * every scrape fetch — including each manually-followed redirect hop — connects
- * only to a validated public address. Returns null if undici can't be loaded;
- * callers then fall back to plain fetch (request-time validation still applies).
+ * only to a validated public address, PAIRED with the fetch implementation that
+ * can actually accept it.
+ *
+ * The pairing is the whole point. A dispatcher is only honoured by the undici
+ * that created it: Node's global `fetch` is backed by the undici BUILT INTO the
+ * runtime (6.21.0 on Node 22, 7.24.4 on Node 25), while this package depends on
+ * undici 8.x. Handing a v8 `Agent` to the global fetch throws
+ * `UND_ERR_INVALID_ARG: invalid onRequestStart method` on every request, on
+ * every supported Node — which is exactly what it did: the entire fetch scrape
+ * path failed instantly, every URL was retried once and then fell back to the
+ * stealth browser (~4.5s per page against ~250ms), and the connect-time SSRF pin
+ * this dispatcher exists for never engaged even once. The browser fallback made
+ * it invisible: scrapes still succeeded, just 15-20x slower.
+ *
+ * Returning the two together makes the invalid combination unrepresentable —
+ * a caller cannot end up holding a dispatcher without the fetch that honours it.
+ * Returns null if undici can't be loaded; callers then use the global fetch with
+ * no dispatcher (request-time validation still applies).
  */
-export async function getSsrfSafeDispatcher(): Promise<unknown | null> {
-  if (cachedSsrfDispatcher !== undefined) return cachedSsrfDispatcher as unknown | null;
+export async function getSsrfSafeFetcher(): Promise<SsrfSafeFetcher | null> {
+  if (cachedSsrfFetcher !== undefined) return cachedSsrfFetcher;
   try {
-    const { Agent } = await import('undici');
-    cachedSsrfDispatcher = new Agent({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent, fetch: undiciFetch } = await import('undici');
+    cachedSsrfFetcher = {
+      fetch: undiciFetch as unknown as SsrfSafeFetcher['fetch'],
+      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } }),
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher as unknown | null;
+  return cachedSsrfFetcher;
 }
 
 /**

package/src/web-research/web-scraper.ts

@@ -23,7 +23,7 @@ import {
 import {
   FETCH_LAYER_TIMEOUT,
 } from './types.ts';
-import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, formatErrorWithCause, isBenignScrapeFailure, } from './scraper-utils.ts';
+import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeFetcher, formatErrorWithCause, isBenignScrapeFailure, isSsrfBlockError, } from './scraper-utils.ts';
 import { isTransientError, abortableDelay } from './retry-utils.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { readBodyCapped, BodyTooLargeError } from '../utils/http-body.ts';
@@ -208,7 +208,7 @@ function decodeHtmlBody(bytes: Uint8Array, contentTypeHeader: string): string {
  * counts are precisely what production incidents are diagnosed from.
  */
 function trackFetchLayerFailure(url: string, error: unknown): void {
-  const isSsrf = error instanceof Error && error.message.includes('not allowed');
+  const isSsrf = isSsrfBlockError(error);
   errorTracker.trackError(error instanceof Error ? error : String(error), {
     component: 'scrapers',
     operation: 'fetch',
@@ -226,7 +226,12 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
   // closing the DNS-rebinding TOCTOU gap between validateUrlForSSRF (request
   // time) and fetch's own re-resolution (connect time). Null when undici is
   // unavailable — request-time validation above still applies.
-  const dispatcher = await getSsrfSafeDispatcher();
+  //
+  // The fetch implementation comes WITH the dispatcher and must be used with it:
+  // a dispatcher is only honoured by the undici that built it, and the global
+  // fetch is backed by Node's bundled undici, not ours. See getSsrfSafeFetcher.
+  const fetcher = await getSsrfSafeFetcher();
+  const doFetch = fetcher?.fetch ?? ((u: string, init: Record<string, unknown>) => fetch(u, init as RequestInit));
 
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), FETCH_LAYER_TIMEOUT);
@@ -257,16 +262,16 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     const userAgent = getRandomUserAgent();
 
     for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-      response = await fetch(currentUrl, {
+      response = await doFetch(currentUrl, {
         signal: controller.signal,
         redirect: 'manual',
         headers: {
           'User-Agent': userAgent,
           'Accept': 'text/html,application/xhtml+xml,application/pdf,*/*;q=0.8',
         },
-        // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.
-        ...(dispatcher ? { dispatcher } : {}),
-      } as RequestInit);
+        // Only ever attached alongside the fetch that honours it (see above).
+        ...(fetcher ? { dispatcher: fetcher.dispatcher } : {}),
+      });
 
       if (response.status >= 300 && response.status < 400) {
         const location = response.headers.get('location');
@@ -380,7 +385,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     // call in a retry, and tracking per attempt would record one failure twice —
     // inflating exactly the error counts these numbers are read to diagnose.
     // scrapeSingle calls trackFetchLayerFailure once with the final error.
-    if (error instanceof Error && error.message.includes('not allowed')) {
+    if (isSsrfBlockError(error)) {
       metrics.increment('scrape_operations_total', 1, { layer: 'fetch', status: 'ssrf_blocked' });
       metrics.observe('scrape_latency_ms', Date.now() - fetchStart, { layer: 'fetch', status: 'ssrf_blocked' });
       throw error;