@lincoln504/pi-research 1.3.9 → 1.3.11

package/README.md

@@ -66,7 +66,7 @@ Three depth levels (normal, deep, ultra) set the team size and number of rounds.
 - **No quota and no monthly fee.** Most AI search providers cap free searches and then charge, with results served from their index on their servers. pi-research searches and scrapes locally through DuckDuckGo in a stealth browser.
 - **You own your index.** Every finding is written to a local [LanceDB](https://lancedb.com) knowledge store that seeds future runs, so repeat questions get faster and cheaper answers.
 - **Cited reports rather than raw dumps.** You get one sourced Markdown report instead of raw web content pasted into the conversation. Context efficient and easy to verify.
-- **Read-only by design.** The research agent cannot run shell commands or write, edit, or delete anything. It can read and search files, so treat it as you would any read-only agent with access to your machine.
+- **Read-only by design.** The research agent cannot run shell commands or write, edit, or delete anything. It can read files, so treat it as you would any read-only agent with access to your machine.
 - **Search a little or a lot.** Depth levels span a quick pass to a large-scale investigation.
 
 ### Requirements / limitations

package/dist/cli.mjs

@@ -2148,13 +2148,18 @@ function isPlausibleCitationUrl(url) {
   }
 }
 function lastCitedLinksHeaderIndex(text) {
+  return lastCitedLinksHeader(text).textStart;
+}
+function lastCitedLinksHeader(text) {
   const re = /(^|\n)([ \t]*(?:#{1,6}[ \t]*|>[ \t]*|\*\*)?)CITED LINKS\b/gi;
-  let last = -1;
+  let lineStart = -1;
+  let textStart = -1;
   let m;
   while ((m = re.exec(text)) !== null) {
-    last = m.index + m[1].length + m[2].length;
+    lineStart = m.index + m[1].length;
+    textStart = m.index + m[1].length + m[2].length;
   }
-  return last;
+  return { lineStart, textStart };
 }
 function parseCitations(report) {
   const headerIdx = lastCitedLinksHeaderIndex(report);
@@ -2889,7 +2894,7 @@ function parseEnvNumber(env, key, def, min, max, integer = false) {
   return n;
 }
 function parseEnvBool(env, key, def) {
-  const v = env[key];
+  const v = env[key]?.trim();
   if (v === void 0 || v === "") return def;
   return v.toLowerCase() === "true";
 }
@@ -2897,7 +2902,7 @@ function parseEnvString(env, key, def) {
   return env[key] || def;
 }
 function parseEnvEnum(env, key, allowed, def) {
-  const raw = env[key];
+  const raw = env[key]?.trim();
   if (raw === void 0 || raw === "") return def;
   if (allowed.includes(raw)) return raw;
   logger.warn(`[config] Invalid ${key}="${raw}" (expected one of: ${allowed.join(", ")}). Using default "${def}".`);
@@ -3049,7 +3054,8 @@ var init_config = __esm({
       /**
        * Comma-separated list of research tools to DISABLE for every researcher in a run
        * (e.g. "scrape", or "youtube_transcript,stackexchange"). Valid tool names:
-       * search, scrape, security_search, stackexchange, youtube_transcript, grep.
+       * search, scrape, security_search, stackexchange, youtube_transcript, grep, read
+       * (RESEARCH_TOOL_NAMES in constants.ts is the canonical list).
        * Disable-only: the list is merged into the per-run excludeTools stream, so disabled
        * tools are both removed from each researcher's toolset AND named in the
        * coordinator/evaluator "DISABLED TOOLS" prompt section. Env-only
@@ -3132,10 +3138,9 @@ function getMaxScrapeBatches(config) {
   }
 }
 function resolveExcludedTools(callerExclusions, configDisabled = []) {
-  const base = callerExclusions && callerExclusions.length > 0 ? callerExclusions : DEFAULT_EXCLUDED_TOOLS;
-  return [.../* @__PURE__ */ new Set([...base, ...configDisabled])];
+  return [.../* @__PURE__ */ new Set([...DEFAULT_EXCLUDED_TOOLS, ...callerExclusions ?? [], ...configDisabled])];
 }
-var MAX_GATHERING_CALLS, MAX_GREP_CALLS, MAX_SCRAPE_URLS, MAX_TEAM_SIZE_LEVEL_1, MAX_TEAM_SIZE_LEVEL_2, MAX_TEAM_SIZE_LEVEL_3, MAX_ROUNDS_LEVEL_1, MAX_ROUNDS_LEVEL_2, MAX_ROUNDS_LEVEL_3, MAX_EXTRA_ROUNDS_WITH_STEERING, OSV_TIMEOUT_MS, WEBGPU_PROBE_TIMEOUT_MS, MAX_QUERY_LENGTH, MAX_FILENAME_QUERY_LENGTH, MIN_QUERY_LENGTH, MAX_EXPORT_RETRIES, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS, DEFAULT_MODEL_CONTEXT_WINDOW, BATCH_2_DEFAULT_CONCURRENCY, MAX_SCRAPE_CONTENT_CHARS_PER_DOC, RESEARCHER_LAUNCH_DELAY_MS, MAX_QUERIES_PER_RESEARCHER_LEVEL_1, MAX_QUERIES_PER_RESEARCHER_LEVEL_2, MAX_QUERIES_PER_RESEARCHER_LEVEL_3, DEFAULT_EXCLUDED_TOOLS;
+var MAX_GATHERING_CALLS, MAX_GREP_CALLS, MAX_SCRAPE_URLS, MAX_TEAM_SIZE_LEVEL_1, MAX_TEAM_SIZE_LEVEL_2, MAX_TEAM_SIZE_LEVEL_3, MAX_ROUNDS_LEVEL_1, MAX_ROUNDS_LEVEL_2, MAX_ROUNDS_LEVEL_3, MAX_EXTRA_ROUNDS_WITH_STEERING, OSV_TIMEOUT_MS, WEBGPU_PROBE_TIMEOUT_MS, MAX_QUERY_LENGTH, MAX_FILENAME_QUERY_LENGTH, MIN_QUERY_LENGTH, MAX_EXPORT_RETRIES, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS, DEFAULT_MODEL_CONTEXT_WINDOW, BATCH_2_DEFAULT_CONCURRENCY, MAX_SCRAPE_CONTENT_CHARS_PER_DOC, RESEARCHER_LAUNCH_DELAY_MS, MAX_QUERIES_PER_RESEARCHER_LEVEL_1, MAX_QUERIES_PER_RESEARCHER_LEVEL_2, MAX_QUERIES_PER_RESEARCHER_LEVEL_3, DEFAULT_EXCLUDED_TOOLS, RESEARCH_TOOL_NAMES;
 var init_constants = __esm({
   "src/constants.ts"() {
     "use strict";
@@ -3167,6 +3172,15 @@ var init_constants = __esm({
     MAX_QUERIES_PER_RESEARCHER_LEVEL_2 = 15;
     MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
     DEFAULT_EXCLUDED_TOOLS = ["grep"];
+    RESEARCH_TOOL_NAMES = [
+      "search",
+      "scrape",
+      "security_search",
+      "stackexchange",
+      "youtube_transcript",
+      "grep",
+      "read"
+    ];
   }
 });
 
@@ -5384,6 +5398,9 @@ var init_store = __esm({
               logger.warn("[store] Failed to read table schema:", schemaErr);
             }
           } else {
+            if (this.options.embedder.getDimension() === null) {
+              await this.withEmbedderReconnect((embedder) => embedder.embedMany([" "]));
+            }
             this.table = await this.createTable();
           }
           await this.evictOldRecords();
@@ -5765,6 +5782,45 @@ var init_store = __esm({
           }
         }
       }
+      /**
+       * Refresh the timestamp of one CURRENT generation for url+type — the
+       * revalidation "touch" for the writer-queue's content-unchanged skip. Without
+       * it a stable page kept its original timestamp forever: with
+       * KNOWLEDGE_STORE_MAX_SERVE_AGE_DAYS set, the serve-age gate then treated the
+       * entry as permanently stale (a live re-scrape on every run, for exactly the
+       * content that never changes), and TTL eviction eventually deleted an entry
+       * that had been re-verified byte-identical possibly hours earlier.
+       */
+      async touchByUrlAndType(url, ingestionType, generationTs, newTs) {
+        return this.trackOperation("touchByUrlAndType", void 0, async () => {
+          let retryCount = 0;
+          const MAX_RETRIES = 5;
+          const BASE_DELAY = 100;
+          const scopeFilter = this.getScopeFilter();
+          while (retryCount <= MAX_RETRIES) {
+            const table = await this.getFreshTable();
+            try {
+              const escapedUrl = url.replace(/'/g, "''");
+              const escapedType = ingestionType.replace(/'/g, "''");
+              await table.update({
+                where: `url = '${escapedUrl}' AND ingestion_type = '${escapedType}' AND timestamp = ${Math.floor(generationTs)} AND (${scopeFilter})`,
+                valuesSql: { timestamp: String(Math.floor(newTs)) }
+              });
+              logger.log(`[store] Refreshed timestamp for unchanged ${ingestionType} chunks of ${url}`);
+              return;
+            } catch (err) {
+              if (isLanceCommitConflict(err)) {
+                retryCount++;
+                if (retryCount > MAX_RETRIES) throw err;
+                const delay2 = Math.floor(Math.random() * (BASE_DELAY * Math.pow(2, retryCount - 1)));
+                await new Promise((r) => setTimeout(r, delay2));
+                continue;
+              }
+              throw err;
+            }
+          }
+        });
+      }
       async findDocumentsByUrl(url) {
         return this.trackOperation("findDocumentsByUrl", [], async () => {
           const table = await this.getFreshTable();
@@ -6002,7 +6058,10 @@ var init_store = __esm({
               backupDirs.map(async (name) => ({ name, ts: await this.backupDirTimestamp(name) }))
             );
             ranked.sort((a, b) => a.ts - b.ts);
-            for (const { name } of ranked.slice(0, -1)) {
+            const ttlDays = this.options.ttlDays ?? 30;
+            const backupCutoff = Math.trunc(Date.now() - ttlDays * 24 * 60 * 60 * 1e3);
+            for (const { name, ts } of ranked.slice(0, -1)) {
+              if (ts >= backupCutoff) continue;
               const fullPath = path13.join(this.options.dbDir, name);
               logger.info(`[store] Removing superseded backup directory: ${name}`);
               await fsPromises.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
@@ -6298,6 +6357,17 @@ var init_writer_queue = __esm({
                 metrics.increment("knowledge_ingest_stale_prune_failed_total", 1);
               });
             }
+            if (this.options.store.touchByUrlAndType) {
+              try {
+                const recheck = (await this.options.store.findByUrl(normalizedUrl)).filter((c) => c.metadata["ingestionType"] === incomingType);
+                const stillNewest = recheck.length > 0 && Math.max(...recheck.map((d) => Number(d.timestamp) || 0)) === newestTs;
+                if (stillNewest) {
+                  await this.options.store.touchByUrlAndType(normalizedUrl, incomingType, newestTs, Date.now());
+                }
+              } catch (err) {
+                logger.warn(`[writer-queue] Timestamp refresh for unchanged ${normalizedUrl} (${incomingType}) failed:`, err);
+              }
+            }
             logger.log(`[writer-queue] Skipping ${normalizedUrl} (${incomingType}) \u2014 content unchanged.`);
             return;
           }
@@ -6776,7 +6846,17 @@ async function loadPipelineWithTimeout(model, device, timeoutMs, useCache) {
         interOpNumThreads: 1
       }
     });
-    return await withTimeout3(pipelinePromise, timeoutMs, errorMessage);
+    let timedOut = true;
+    try {
+      const p = await withTimeout3(pipelinePromise, timeoutMs, errorMessage);
+      timedOut = false;
+      return p;
+    } finally {
+      if (timedOut) {
+        pipelinePromise.then((late) => late.dispose?.()).catch(() => {
+        });
+      }
+    }
   });
   return { pipeline: loadedPipeline, errorMessage };
 }
@@ -10421,7 +10501,7 @@ function normalizeCitations(reports) {
         localToGlobal.set(localKey, globalId);
       }
     });
-    const headerIdx = lastCitedLinksHeaderIndex(report);
+    const headerIdx = lastCitedLinksHeader(report).lineStart;
     let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
     content = rewriteCitationMarkers(
       content,
@@ -17024,17 +17104,30 @@ function ssrfSafeLookup(hostname, options, callback, resolveImpl = dnsCb.lookup)
     return callback(null, safe[0].address, safe[0].family);
   });
 }
-var cachedSsrfDispatcher = void 0;
-async function getSsrfSafeDispatcher() {
-  if (cachedSsrfDispatcher !== void 0) return cachedSsrfDispatcher;
+function isSsrfBlockError(error) {
+  let e = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if (e.code === "ESSRFBLOCKED" || e.message.includes("not allowed")) {
+      return true;
+    }
+    e = e.cause;
+  }
+  return false;
+}
+var cachedSsrfFetcher = void 0;
+async function getSsrfSafeFetcher() {
+  if (cachedSsrfFetcher !== void 0) return cachedSsrfFetcher;
   try {
-    const { Agent: Agent2 } = await import("undici");
-    cachedSsrfDispatcher = new Agent2({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent: Agent2, fetch: undiciFetch } = await import("undici");
+    cachedSsrfFetcher = {
+      fetch: undiciFetch,
+      dispatcher: new Agent2({ connect: { lookup: ssrfSafeLookup } })
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher;
+  return cachedSsrfFetcher;
 }
 function formatErrorWithCause(err) {
   try {
@@ -17284,7 +17377,7 @@ function decodeHtmlBody(bytes, contentTypeHeader) {
   }
 }
 function trackFetchLayerFailure(url, error) {
-  const isSsrf = error instanceof Error && error.message.includes("not allowed");
+  const isSsrf = isSsrfBlockError(error);
   errorTracker.trackError(error instanceof Error ? error : String(error), {
     component: "scrapers",
     operation: "fetch",
@@ -17296,7 +17389,8 @@ function trackFetchLayerFailure(url, error) {
 }
 async function scrapeWithFetch(url, signal) {
   await validateUrlForSSRF(url);
-  const dispatcher = await getSsrfSafeDispatcher();
+  const fetcher = await getSsrfSafeFetcher();
+  const doFetch = fetcher?.fetch ?? ((u, init) => fetch(u, init));
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), FETCH_LAYER_TIMEOUT);
   safeUnref(timeoutId);
@@ -17315,15 +17409,15 @@ async function scrapeWithFetch(url, signal) {
     let response;
     const userAgent = getRandomUserAgent();
     for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-      response = await fetch(currentUrl, {
+      response = await doFetch(currentUrl, {
         signal: controller.signal,
         redirect: "manual",
         headers: {
           "User-Agent": userAgent,
           "Accept": "text/html,application/xhtml+xml,application/pdf,*/*;q=0.8"
         },
-        // RequestInit's standard type omits undici's `dispatcher`; cast to attach it.
-        ...dispatcher ? { dispatcher } : {}
+        // Only ever attached alongside the fetch that honours it (see above).
+        ...fetcher ? { dispatcher: fetcher.dispatcher } : {}
       });
       if (response.status >= 300 && response.status < 400) {
         const location = response.headers.get("location");
@@ -17424,7 +17518,7 @@ async function scrapeWithFetch(url, signal) {
     metrics.observe("scrape_latency_ms", fetchDuration, { layer: "fetch", content_type: "html", status: "success" });
     return { source: "fetch", layer: "fetch", markdown };
   } catch (error) {
-    if (error instanceof Error && error.message.includes("not allowed")) {
+    if (isSsrfBlockError(error)) {
       metrics.increment("scrape_operations_total", 1, { layer: "fetch", status: "ssrf_blocked" });
       metrics.observe("scrape_latency_ms", Date.now() - fetchStart, { layer: "fetch", status: "ssrf_blocked" });
       throw error;
@@ -21618,7 +21712,7 @@ async function createResearcherSession(options) {
 
 // src/orchestration/researcher-executor.ts
 init_text_utils();
-init_llm();
+init_llm_usage();
 init_logger();
 init_metrics();
 init_service_interfaces();
@@ -22054,12 +22148,13 @@ ${prompt}`);
         if (msg2?.["role"] !== "assistant") return;
         const rawUsage = msg2["usage"];
         if (rawUsage) {
-          const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
+          const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+            component: "researcher",
+            complexity,
+            observer: { onTokensConsumed: (t, c) => safeObserve(() => observer?.onTokensConsumed?.(t, c)) }
+          });
           if (tokens > 0 || cost > 0) {
-            metrics.increment("llm_tokens_total", tokens, { component: "researcher", complexity: String(complexity) });
-            metrics.increment("llm_cost_total", cost, { component: "researcher", complexity: String(complexity) });
             safeObserve(() => observer?.onResearcherProgress?.(id, void 0, tokens, cost));
-            safeObserve(() => observer?.onTokensConsumed?.(tokens, cost));
           }
         }
       } else if (event.type === "tool_execution_start") {
@@ -22186,7 +22281,7 @@ ${responseText}`);
 
 // src/orchestration/quick-research-orchestrator.ts
 init_prompts();
-init_llm();
+init_llm_usage();
 init_logger();
 init_config();
 init_text_utils();
@@ -22467,12 +22562,13 @@ ${prompt}`);
           if (msg2?.["role"] !== "assistant") return;
           const rawUsage = msg2["usage"];
           if (rawUsage) {
-            const { tokens, cost } = extractUsage(resolvedModel, rawUsage);
+            const { tokens, cost } = recordLlmUsage(resolvedModel, rawUsage, {
+              component: "quick_researcher",
+              complexity: 0,
+              observer
+            });
             if (tokens > 0 || cost > 0) {
-              metrics.increment("llm_tokens_total", tokens, { component: "quick_researcher", complexity: "0" });
-              metrics.increment("llm_cost_total", cost, { component: "quick_researcher", complexity: "0" });
               observer?.onResearcherProgress?.("quick", void 0, tokens, cost);
-              observer?.onTokensConsumed?.(tokens, cost);
             }
           }
         } else if (event.type === "tool_execution_start") {
@@ -23075,6 +23171,13 @@ var ResearchOrchestrationService = class {
       });
       logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
     } catch (err) {
+      const isTerminal = err instanceof ResearchRunCapacityError || err instanceof Error && err.name === "AbortError";
+      if (isTerminal) {
+        try {
+          observer?.onError?.(err);
+        } catch {
+        }
+      }
       if (err instanceof ResearchRunCapacityError) throw err;
       if (err instanceof Error && err.name === "AbortError") throw err;
       logger.warn(`[ResearchOrchestrationService] Run-cap unavailable, proceeding without it: ${err instanceof Error ? err.message : String(err)}`);
@@ -23826,7 +23929,7 @@ Research performed using ${modelId}`;
           if (globalId !== void 0) localToGlobal.set(localKey, globalId);
           else droppedNumbers.add(localKey);
         });
-        let body = synthesis.slice(0, citedHeaderIdx).trimEnd();
+        let body = synthesis.slice(0, lastCitedLinksHeader(synthesis).lineStart).trimEnd();
         if (localToGlobal.size > 0 || droppedNumbers.size > 0) {
           body = rewriteCitationMarkers(body, localToGlobal, (n) => droppedNumbers.has(n));
         }
@@ -24749,6 +24852,7 @@ Total cost: ${costStr}`;
 // src/cli.ts
 init_config();
 init_model_registry_factory();
+init_constants();
 import { getAgentDir as getAgentDir2 } from "@earendil-works/pi-coding-agent";
 function readPkgVersion() {
   try {
@@ -24869,14 +24973,15 @@ function makeProgressObserver() {
 var ConfigFileError = class extends Error {
 };
 function bridgeConfigEnv(explicitConfigPath) {
-  const basePath = explicitConfigPath ?? getGlobalEnvFilePath();
+  const globalBasePath = getGlobalEnvFilePath();
+  const basePath = explicitConfigPath ?? globalBasePath;
   const cliOverlayPath = getInterfaceEnvFilePath("cli");
   let loaded = false;
   const merged = {};
   const fromExplicitBase = /* @__PURE__ */ new Set();
-  const layers = explicitConfigPath !== void 0 ? [cliOverlayPath, basePath] : [basePath, cliOverlayPath];
+  const layers = explicitConfigPath !== void 0 ? [globalBasePath, cliOverlayPath, explicitConfigPath] : [globalBasePath, cliOverlayPath];
   for (const filePath of layers) {
-    const isExplicit = explicitConfigPath !== void 0 && filePath === basePath;
+    const isExplicit = explicitConfigPath !== void 0 && filePath === explicitConfigPath;
     try {
       if (!fs17.existsSync(filePath)) {
         if (isExplicit) throw new ConfigFileError(`--config file not found: ${filePath}`);
@@ -25013,7 +25118,11 @@ async function detectCredentials(explicitModel) {
     modelFrom,
     piAuthPresent,
     piModelsPresent,
-    problem: "No model or API key is configured for pi-research. Provide credentials via environment variables (PI_RESEARCH_API_KEY + PI_RESEARCH_PROVIDER + PI_RESEARCH_MODEL) or, if you use pi, via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider apiKey in ~/.pi/agent/models.json)."
+    // A model configured with no usable key is a different (and misleading to
+    // conflate) gap than nothing configured at all: SKILL.md tells agents to relay
+    // this text verbatim, so "No model … is configured" pointed users who HAD set
+    // PI_RESEARCH_MODEL at the wrong knob.
+    problem: model ? `A model is configured (${model}) but no API key is available for ${provider ? `provider "${provider}"` : "its provider"}. Provide one via the PI_RESEARCH_API_KEY environment variable (with PI_RESEARCH_PROVIDER), or, if you use pi, via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider apiKey in ~/.pi/agent/models.json).` : "No model or API key is configured for pi-research. Provide credentials via environment variables (PI_RESEARCH_API_KEY + PI_RESEARCH_PROVIDER + PI_RESEARCH_MODEL) or, if you use pi, via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider apiKey in ~/.pi/agent/models.json)."
   };
 }
 function configBlock(det, extraNote) {
@@ -25083,6 +25192,8 @@ Error: ${vmsg}
       ...args.excludeTools ? { excludeTools: args.excludeTools } : {},
       ...args.initialLinks ? { initialLinks: args.initialLinks } : {}
     }, abortController.signal);
+    const outcome = getLastResearcherOutcome();
+    const errorReport = getLastErrorReport();
     let reportPath = null;
     const exportCfg = getConfig(process.cwd(), "cli");
     if (exportCfg.RESEARCH_REPORT_EXPORT_ENABLED) {
@@ -25097,8 +25208,6 @@ Error: ${vmsg}
       exit = cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED;
     }
     if (args.json) {
-      const outcome = getLastResearcherOutcome();
-      const errorReport = getLastErrorReport();
       toStdout(pretty({
         ok: true,
         depth,
@@ -25175,8 +25284,11 @@ Configure in:
     if (activeResearchAbortController === abortController) activeResearchAbortController = null;
     await safeShutdown();
   }
+  if (cancellationRequested) {
+    exit = cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED;
+  }
   if (json) {
-    toStdout(pretty({ ok: true, ...result }));
+    toStdout(pretty({ ok: true, ...cancellationRequested ? { cancelled: true } : {}, ...result }));
   } else {
     toStdout(result.text.endsWith("\n") ? result.text : result.text + "\n");
   }
@@ -25511,6 +25623,7 @@ COMMANDS
     --dry-run | --json           Plan only; JSON output.
 
   help, --help, -h               Show this help.
+  --version, -v                  Print the pi-research version.
 
 CONFIGURE
   Credentials are resolved in this order (first match wins):
@@ -25700,6 +25813,12 @@ function parseArgs(argv) {
       } else if (a === "--exclude-tools") {
         const v = takeOptionValue(rest, ++i, "--exclude-tools");
         excludeTools = v.split(",").map((s) => s.trim()).filter(Boolean);
+        const unknown = excludeTools.filter((t) => !RESEARCH_TOOL_NAMES.includes(t));
+        if (unknown.length > 0) {
+          throw new UsageError(
+            `--exclude-tools: unknown tool name${unknown.length === 1 ? "" : "s"} ${unknown.map((t) => `"${t}"`).join(", ")}. Valid names: ${RESEARCH_TOOL_NAMES.join(", ")}.`
+          );
+        }
         if (excludeTools.length === 0) excludeTools = void 0;
       } else if (a === "--initial-links") {
         let j = i + 1;
@@ -25713,6 +25832,7 @@ function parseArgs(argv) {
           }
           j++;
         }
+        if (j === i + 1) throw new UsageError("--initial-links requires at least one URL.");
         i = j - 1;
       } else if (a === "--config") {
         configPath = takeOptionValue(rest, ++i, "--config");

package/dist/thread-worker.mjs

@@ -1512,8 +1512,9 @@ __export(scraper_utils_exports, {
   extractDomain: () => extractDomain,
   formatErrorWithCause: () => formatErrorWithCause,
   getRandomUserAgent: () => getRandomUserAgent,
-  getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
+  getSsrfSafeFetcher: () => getSsrfSafeFetcher,
   isBenignScrapeFailure: () => isBenignScrapeFailure,
+  isSsrfBlockError: () => isSsrfBlockError,
   ssrfSafeLookup: () => ssrfSafeLookup,
   stripImageLinks: () => stripImageLinks,
   validateContent: () => validateContent,
@@ -1633,16 +1634,29 @@ function ssrfSafeLookup(hostname, options, callback, resolveImpl = dnsCb.lookup)
     return callback(null, safe[0].address, safe[0].family);
   });
 }
-async function getSsrfSafeDispatcher() {
-  if (cachedSsrfDispatcher !== void 0) return cachedSsrfDispatcher;
+function isSsrfBlockError(error) {
+  let e = error;
+  for (let depth = 0; depth < 5 && e instanceof Error; depth++) {
+    if (e.code === "ESSRFBLOCKED" || e.message.includes("not allowed")) {
+      return true;
+    }
+    e = e.cause;
+  }
+  return false;
+}
+async function getSsrfSafeFetcher() {
+  if (cachedSsrfFetcher !== void 0) return cachedSsrfFetcher;
   try {
-    const { Agent } = await import("undici");
-    cachedSsrfDispatcher = new Agent({ connect: { lookup: ssrfSafeLookup } });
+    const { Agent, fetch: undiciFetch } = await import("undici");
+    cachedSsrfFetcher = {
+      fetch: undiciFetch,
+      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
+    };
   } catch (e) {
     logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfDispatcher = null;
+    cachedSsrfFetcher = null;
   }
-  return cachedSsrfDispatcher;
+  return cachedSsrfFetcher;
 }
 function formatErrorWithCause(err) {
   try {
@@ -1776,7 +1790,7 @@ function validateContent(html, markdown, url) {
     throw error;
   }
 }
-var cachedSsrfDispatcher;
+var cachedSsrfFetcher;
 var init_scraper_utils = __esm({
   "src/web-research/scraper-utils.ts"() {
     "use strict";
@@ -1785,7 +1799,7 @@ var init_scraper_utils = __esm({
     init_logger();
     init_scraper_types();
     init_browser_error_utils();
-    cachedSsrfDispatcher = void 0;
+    cachedSsrfFetcher = void 0;
   }
 });
 
@@ -2566,7 +2580,8 @@ var ConfigSchema = Type.Object({
   /**
    * Comma-separated list of research tools to DISABLE for every researcher in a run
    * (e.g. "scrape", or "youtube_transcript,stackexchange"). Valid tool names:
-   * search, scrape, security_search, stackexchange, youtube_transcript, grep.
+   * search, scrape, security_search, stackexchange, youtube_transcript, grep, read
+   * (RESEARCH_TOOL_NAMES in constants.ts is the canonical list).
    * Disable-only: the list is merged into the per-run excludeTools stream, so disabled
    * tools are both removed from each researcher's toolset AND named in the
    * coordinator/evaluator "DISABLED TOOLS" prompt section. Env-only

package/docs/ARCHITECTURE.md

@@ -12,7 +12,7 @@ pi CLI
 └── pi-research extension (src/index.ts)
     ├── Registered Tools   research, health, research_knowledge_search (always registered; reports why when the store is disabled)
     ├── Commands           /research, /research-config, /knowledge-store
-    ├── Events             input (mid-run steering), session_shutdown (cleanup)
+    ├── Events             input (mid-run steering), session_shutdown (cleanup), session_before_compact / session_compact, before_agent_start, after_provider_response
     └── Layers
         ├── Orchestration   quick/deep research coordination
         ├── Agent Tools     search, scrape, youtube_transcript, security_search, stackexchange, grep, read
@@ -106,21 +106,23 @@ agent to invoke:
 |------|-------|------|---------|
 | `search` | ✓ | — | DuckDuckGo Lite via the stealth browser |
 | `scrape` | ✓ | ✓ | Batch page fetch → Markdown via the stealth browser (up to 6 URLs per call) |
-| `youtube_transcript` | ✓ | ✓ | YouTube captions via youtubei.js + BotGuard PoToken (≤3 videos, one call per researcher) |
+| `youtube_transcript` | ✓ | ✓ | YouTube captions via youtubei.js + BotGuard PoToken (≤3 videos by default, configurable 1–5; one call per researcher) |
 | `security_search` | ✓ | ✓ | NVD, CISA KEV, GitHub Advisories, OSV |
 | `stackexchange` | ✓ | ✓ | Stack Exchange network |
-| `grep` | — | — | Local ripgrep (from pi-coding-agent) — off by default, see below |
+| `grep` | — | — | Local ripgrep (from pi-coding-agent) — always excluded, see below |
 | `read` | ✓ | ✓ | Local file reads (from pi-coding-agent) |
 
 In deep research `search` is excluded — the coordinator runs the search burst and hands
 out URLs directly.
 
-`grep` is excluded by default at **every** depth and on every front-end (CLI, SDK, agent
-skill, pi extension): this is web research, and a capable model otherwise spends turns
-searching the local filesystem. Passing an explicit `excludeTools` list — `--exclude-tools`
-on the CLI, the `excludeTools` tool parameter in the extension — REPLACES that default, so
-naming other tools re-enables `grep`. `PI_RESEARCH_DISABLED_TOOLS` is separate and strictly
-additive: it can only ever remove capabilities (see [CONFIGURATION.md](CONFIGURATION.md)).
+`grep` is excluded at **every** depth and on every front-end (CLI, SDK, agent skill,
+pi extension): this is web research, and a capable model otherwise spends turns
+searching the local filesystem. Both exclusion surfaces — the `excludeTools` list
+(`--exclude-tools` on the CLI, the `excludeTools` tool parameter in the extension) and
+`PI_RESEARCH_DISABLED_TOOLS` — are strictly additive on top of that default: they can
+only ever remove capabilities (see [CONFIGURATION.md](CONFIGURATION.md)). Before 1.3.10
+a non-empty `excludeTools` list replaced the default, so naming any other tool silently
+re-enabled `grep`.
 
 Researchers cannot write files, run shell commands, or reach the network outside these
 tools.
@@ -288,11 +290,12 @@ src/
 ### Key design decisions
 
 Read-only researchers — researcher agents are limited to the tool set above. They cannot
-write files, spawn processes, or make arbitrary network calls. They *can* read and grep
-files: `read` and `grep` are registered unconditionally and the researcher exclusion list
-(`bash`, `write`, `edit`, `repl`, `git`, `terminal`) does not cover them. The `cwd` passed
-to those tools is a resolution base, not a jail — an absolute path resolves to itself — so
-the boundary is "no mutation", not "only this directory".
+write files, spawn processes, or make arbitrary network calls. They *can* read files:
+`read` is registered and the researcher exclusion list (`bash`, `write`, `edit`, `repl`,
+`git`, `terminal`) does not cover it. Local `grep` is registered but always excluded (see
+the tool table above). The `cwd` passed to `read` is a resolution base, not a jail — an
+absolute path resolves to itself — so the boundary is "no mutation", not "only this
+directory".
 
 Worker pool over direct browser — browser processes are isolated in workers so a crash
 in one cannot affect the orchestrator or other sessions.