@lincoln504/pi-research 1.6.8 → 1.6.10

package/README.md

@@ -53,6 +53,10 @@ tries to compile from source; upgrading npm fixes it.
 
 In pi it works out of the box on the session's model and pi's configuration. Standalone use ([agent skill](docs/AGENT-SKILL.md) or [SDK](docs/SDK.md)) needs a model configured. See [Configuration](docs/CONFIGURATION.md).
 
+### Uninstall
+
+`pi remove npm:@lincoln504/pi-research` removes the extension, and `npm uninstall -g @lincoln504/pi-research` the standalone engine. npm 7+ no longer runs `preuninstall`, so **nothing else is removed on its own**: skill links into other agents (Claude Code, Codex, …), the state directory (`~/.pi/research/state`), and the cache (`~/.cache/pi-research`, including any downloaded embedding models) stay in place. Remove the skill links first with `pi-research skill uninstall` (or `/research-config` → Remove from External Agents); the shared stealth-browser cache (`~/.cache/camoufox`) is preserved unless `PI_RESEARCH_PURGE_BROWSERS=1`. See [AGENT-SKILL.md](docs/AGENT-SKILL.md#installation-flow) for the full picture.
+
 ### How it works
 
 A research run loops through agent teams: a coordinator plans and starts with a search, researcher agents scrape and read in parallel, and a research lead decides whether to go another round, then writes the report from every report collected. The result is one cited Markdown report with findings optionally saved to the knowledge store.

package/agent-skill/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.6.8", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.6.10", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---
 

package/dist/cli.mjs

@@ -2387,6 +2387,46 @@ var init_atomic_replace = __esm({
   }
 });
 
+// src/knowledge/availability.ts
+import { createRequire as createRequire3 } from "node:module";
+function probeKnowledgeStoreAvailability(resolver = defaultResolver) {
+  if (cachedIsValidFor(resolver)) return _cached;
+  const missing = [];
+  for (const pkg of REQUIRED_PACKAGES) {
+    try {
+      resolver(pkg);
+    } catch {
+      missing.push(pkg);
+    }
+  }
+  const result = { available: missing.length === 0, missing };
+  if (resolver === defaultResolver) _cached = result;
+  return result;
+}
+function cachedIsValidFor(resolver) {
+  return _cached !== null && resolver === defaultResolver;
+}
+function describeKnowledgeStoreUnavailability(availability) {
+  if (availability.available) return "";
+  const parts = availability.missing.map(
+    (pkg) => pkg === "@huggingface/transformers" ? `'${pkg}' (optional embedding dependency \u2014 likely skipped at install time)` : `'${pkg}' (broken install \u2014 native vector store missing)`
+  );
+  return `knowledge store is OFF: required package${parts.length > 1 ? "s" : ""} not resolvable \u2014 ${parts.join("; ")}. Repair with a full install of optional dependencies (e.g. 'npm install' including optional deps, or pi install npm:@lincoln504/pi-research without --omit=optional).`;
+}
+var REQUIRED_PACKAGES, _cached, defaultResolver;
+var init_availability = __esm({
+  "src/knowledge/availability.ts"() {
+    "use strict";
+    REQUIRED_PACKAGES = ["@lancedb/lancedb", "@huggingface/transformers"];
+    _cached = null;
+    defaultResolver = (specifier) => (
+      // import.meta.url survives esbuild's ESM output; createRequire only builds a
+      // resolver — it never evaluates the specifier.
+      createRequire3(import.meta.url).resolve(specifier)
+    );
+  }
+});
+
 // src/config.ts
 import * as fs5 from "node:fs";
 import * as path5 from "node:path";
@@ -2593,7 +2633,19 @@ function describeKnowledgeStoreMode(cwd = process.cwd(), iface) {
     else if (iface && fileHasKey(getInterfaceEnvFilePath(iface))) origin = `${iface}.env overlay`;
     else if (fileHasKey(getGlobalEnvFilePath())) origin = "config.env (machine-wide default)";
   }
-  return { mode: cfg.KNOWLEDGE_STORE_MODE, origin, dbDir: getDbDir(cfg, cwd) };
+  return {
+    mode: cfg.KNOWLEDGE_STORE_MODE,
+    origin,
+    dbDir: getDbDir(cfg, cwd),
+    ...availabilityFields()
+  };
+}
+function availabilityFields() {
+  const availability = probeKnowledgeStoreAvailability();
+  return {
+    available: availability.available,
+    unavailableReason: describeKnowledgeStoreUnavailability(availability)
+  };
 }
 function parseDotEnv(content) {
   const out = {};
@@ -2744,6 +2796,7 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
     PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: String(config.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING),
     PI_RESEARCH_AVG_TOKENS_PER_SCRAPE: String(config.AVG_TOKENS_PER_SCRAPE),
     PI_RESEARCH_MAX_CONCURRENT_SCRAPES: String(config.MAX_CONCURRENT_SCRAPES),
+    PI_RESEARCH_MAX_SCRAPE_URLS: String(config.MAX_SCRAPE_URLS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS: String(config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS: String(config.YOUTUBE_TRANSCRIPT_TIMEOUT_MS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG: config.YOUTUBE_TRANSCRIPT_LANG,
@@ -2905,6 +2958,7 @@ function createConfig(env, processEnv) {
     MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: parseEnvNumber(e, "PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING", DEFAULTS.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING, 0.05, 1),
     AVG_TOKENS_PER_SCRAPE: parseEnvNumber(e, "PI_RESEARCH_AVG_TOKENS_PER_SCRAPE", DEFAULTS.AVG_TOKENS_PER_SCRAPE, 500, 1e4),
     MAX_CONCURRENT_SCRAPES: parseEnvNumber(e, "PI_RESEARCH_MAX_CONCURRENT_SCRAPES", DEFAULTS.MAX_CONCURRENT_SCRAPES, 1, 20, true),
+    MAX_SCRAPE_URLS: parseEnvNumber(e, "PI_RESEARCH_MAX_SCRAPE_URLS", DEFAULTS.MAX_SCRAPE_URLS, 1, 20, true),
     YOUTUBE_TRANSCRIPT_MAX_VIDEOS: parseEnvNumber(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS", DEFAULTS.YOUTUBE_TRANSCRIPT_MAX_VIDEOS, 1, 5, true),
     YOUTUBE_TRANSCRIPT_TIMEOUT_MS: parseEnvNumber(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS", DEFAULTS.YOUTUBE_TRANSCRIPT_TIMEOUT_MS, 5e3, 12e4),
     YOUTUBE_TRANSCRIPT_LANG: parseEnvString(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG", DEFAULTS.YOUTUBE_TRANSCRIPT_LANG),
@@ -3002,6 +3056,7 @@ var init_config = __esm({
     init_host_config();
     init_text_utils();
     init_atomic_replace();
+    init_availability();
     ConfigSchema = Type.Object({
       /** Per-researcher timeout in milliseconds (default: 300000, range: 3-30 min) */
       RESEARCHER_TIMEOUT_MS: Type.Number({ minimum: 18e4, maximum: 18e5, default: 3e5 }),
@@ -3046,8 +3101,10 @@ var init_config = __esm({
       MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1, default: 0.15 }),
       /** Estimated tokens per scrape result for planning (default: 2500) */
       AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 1e4, default: 2500 }),
-      /** Maximum number of concurrent scrapes (default: 3) */
+      /** Maximum concurrent scrapes (default: 3) */
       MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+      /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+      MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
       /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
       YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
       /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -3184,6 +3241,7 @@ var init_config = __esm({
       "PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING",
       "PI_RESEARCH_AVG_TOKENS_PER_SCRAPE",
       "PI_RESEARCH_MAX_CONCURRENT_SCRAPES",
+      "PI_RESEARCH_MAX_SCRAPE_URLS",
       "PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS",
       "PI_RESEARCH_SEARCH_TIMEOUT_MS",
       "PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS",
@@ -3240,6 +3298,13 @@ function getMaxScrapeBatches(config, cachingActive = false) {
     return 2;
   }
 }
+function getMaxScrapeUrls(config) {
+  try {
+    return (config || getConfig()).MAX_SCRAPE_URLS;
+  } catch {
+    return MAX_SCRAPE_URLS;
+  }
+}
 function resolveExcludedTools(callerExclusions, configDisabled = []) {
   return [.../* @__PURE__ */ new Set([...DEFAULT_EXCLUDED_TOOLS, ...callerExclusions ?? [], ...configDisabled])];
 }
@@ -3250,7 +3315,7 @@ var init_constants = __esm({
     init_config();
     MAX_GATHERING_CALLS = 12;
     MAX_GREP_CALLS = 30;
-    MAX_SCRAPE_URLS = 6;
+    MAX_SCRAPE_URLS = 8;
     MAX_TEAM_SIZE_LEVEL_1 = 2;
     MAX_TEAM_SIZE_LEVEL_2 = 3;
     MAX_TEAM_SIZE_LEVEL_3 = 5;
@@ -9007,11 +9072,15 @@ function getMigrationStrategy(config) {
   logger.warn(`[knowledge] Invalid migration strategy '${strategy}'. Valid options: drop, re-embed, backup. Falling back to default (backup).`);
   return void 0;
 }
-async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory, withLock, config = getConfig(), workspace = process.cwd()) {
+async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory, withLock, config = getConfig(), workspace = process.cwd(), availability = probeKnowledgeStoreAvailability()) {
   if (config.KNOWLEDGE_STORE_MODE === "none") {
     logger.debug("[knowledge] Knowledge store is disabled in configuration");
     return null;
   }
+  if (!availability.available) {
+    logger.warn(`[knowledge] ${describeKnowledgeStoreUnavailability(availability)}`);
+    return null;
+  }
   validateConfig(config);
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {
     let store;
@@ -9089,6 +9158,7 @@ var init_knowledge = __esm({
     init_config();
     init_logger();
     init_model_config();
+    init_availability();
     init_model_config();
     MAX_INIT_RETRIES = 5;
     BASE_RETRY_DELAY_MS = 1e3;
@@ -9115,6 +9185,7 @@ var init_knowledge_store_service = __esm({
     init_embedding_factory();
     init_config();
     init_embedder_utils();
+    init_availability();
     KnowledgeStoreService = class {
       name = ServiceNames.KNOWLEDGE_STORE;
       lifecycle = "uninitialized" /* UNINITIALIZED */;
@@ -9255,9 +9326,10 @@ var init_knowledge_store_service = __esm({
               }
             }
             if (!components) {
-              logger.debug("[KnowledgeStoreService] Knowledge store is disabled. Setting lifecycle to DISABLED.");
+              const depsMissing = !probeKnowledgeStoreAvailability().available;
+              logger.debug(`[KnowledgeStoreService] Knowledge store is disabled (${depsMissing ? "dependencies missing" : "mode = none"}). Setting lifecycle to DISABLED.`);
               this.lifecycle = "disabled" /* DISABLED */;
-              this._disabledReason = "mode";
+              this._disabledReason = depsMissing ? "native" : "mode";
               this._embedder = null;
               this._store = null;
               this._writerQueue = null;
@@ -10104,6 +10176,9 @@ function missResult(reason) {
     case "store_disabled":
       text = "No results found (knowledge store is disabled in settings). Live research can get the info.";
       break;
+    case "store_packages_missing":
+      text = "No results found (knowledge store is off \u2014 its required packages are not installed on this host, so it stays off regardless of settings). Live research can get the info.";
+      break;
     case "store_not_ready":
       text = "No results found (knowledge store is initializing). Live research can get the info.";
       break;
@@ -10166,8 +10241,10 @@ function createResearchKnowledgeSearchTool(iface) {
         }
         const store = await storeService.getStore();
         if (!store) {
-          const lifecycle = storeService.lifecycle;
-          return missResult(lifecycle === "disabled" ? "store_disabled" : "store_not_ready");
+          const disabledReason = storeService.getDisabledReason();
+          if (disabledReason === "native") return missResult("store_packages_missing");
+          if (disabledReason === "mode") return missResult("store_disabled");
+          return missResult("store_not_ready");
         }
         const count = await store.count();
         if (count === 0) {
@@ -12537,7 +12614,7 @@ import * as crypto4 from "node:crypto";
 import { join as join8 } from "node:path";
 import { existsSync as existsSync6, mkdirSync as mkdirSync3 } from "node:fs";
 import { platform, homedir as homedir3 } from "node:os";
-import { createRequire as createRequire3 } from "node:module";
+import { createRequire as createRequire4 } from "node:module";
 function getWindowsCamoufoxDir() {
   return join8(homedir3(), "AppData", "Local", "camoufox", "camoufox", "Cache");
 }
@@ -12655,7 +12732,7 @@ function probeBrowserNativeDeps(load = defaultNativeDepLoad) {
   }
 }
 function defaultNativeDepLoad() {
-  const require2 = createRequire3(import.meta.url);
+  const require2 = createRequire4(import.meta.url);
   const Database = require2("better-sqlite3");
   const db = new Database(":memory:");
   db.close();
@@ -15988,7 +16065,7 @@ import { FixedClusterPool, WorkerChoiceStrategies } from "poolifier";
 // src/infrastructure/browser/ensure-browser.ts
 init_logger();
 import { spawn as spawn2 } from "node:child_process";
-import { createRequire as createRequire4 } from "node:module";
+import { createRequire as createRequire5 } from "node:module";
 import { createHash as createHash2 } from "node:crypto";
 import { existsSync as existsSync7, readdirSync as readdirSync3, statSync as statSync3, fstatSync, openSync as openSync4, closeSync as closeSync4, rmSync } from "node:fs";
 import { join as join11, dirname as dirname8 } from "node:path";
@@ -16015,7 +16092,7 @@ function isBrowserBinaryPresent() {
 }
 function resolveCamoufoxBin() {
   try {
-    const require2 = createRequire4(import.meta.url);
+    const require2 = createRequire5(import.meta.url);
     const pkgJson = require2.resolve("camoufox-js/package.json");
     const bin = join11(dirname8(pkgJson), "..", ".bin", process.platform === "win32" ? "camoufox-js.cmd" : "camoufox-js");
     if (existsSync7(bin)) return bin;
@@ -17939,6 +18016,7 @@ healthRegistry.initialize().catch(() => {
 
 // src/healthcheck/index.ts
 init_embedder_utils();
+init_availability();
 async function checkBrowserCapability() {
   const mockMode = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" && process.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
   if (isBrowserAvailable() || mockMode) {
@@ -17957,6 +18035,11 @@ async function checkBrowserCapability() {
     return { healthy: false, error: 'Camoufox (browser) not found. Run "npx camoufox-js fetch" to install the browser.' };
   }
 }
+function knowledgeStoreDisabledStatus() {
+  const availability = probeKnowledgeStoreAvailability();
+  const detail = availability.available ? "native embedding/vector stack unavailable on this platform" : `required packages not resolvable \u2014 ${availability.missing.join(", ")}`;
+  return `disabled (${detail})`;
+}
 function registerHealthChecks(registry, container = getServiceContainer()) {
   const config = getConfig();
   const healthTimeoutMs = config.HEALTH_CHECK_TIMEOUT_MS;
@@ -17988,7 +18071,7 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       if (existing?.lifecycle === "disabled" /* DISABLED */) {
         const reason = existing.getDisabledReason();
         return { healthy: true, diagnostic: {
-          status: reason === "native" ? "disabled (native embedding/vector stack unavailable on this platform)" : "disabled"
+          status: reason === "native" ? knowledgeStoreDisabledStatus() : "disabled"
         } };
       }
       const status = existing?.lifecycle === "initializing" /* INITIALIZING */ ? "initializing" : existing?.lifecycle === "disposed" /* DISPOSED */ || existing?.lifecycle === "disposing" /* DISPOSING */ ? "disposed (not running)" : "ready (idle)";
@@ -18007,12 +18090,12 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       return { healthy: true, diagnostic: { status: "disabled in config" } };
     }
     if (isNativeStackUnavailableError(initError)) {
-      return { healthy: true, diagnostic: { status: "disabled (native embedding/vector stack unavailable on this platform)" } };
+      return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
     }
     if (service?.lifecycle === "disabled" /* DISABLED */) {
       const reason = service.getDisabledReason();
       return { healthy: true, diagnostic: {
-        status: reason === "native" ? "disabled (native embedding/vector stack unavailable on this platform)" : "disabled"
+        status: reason === "native" ? knowledgeStoreDisabledStatus() : "disabled"
       } };
     }
     try {
@@ -18061,7 +18144,7 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       };
     } catch (e) {
       if (isNativeStackUnavailableError(e)) {
-        return { healthy: true, diagnostic: { status: "disabled (native embedding/vector stack unavailable on this platform)" } };
+        return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
       }
       return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
     }
@@ -19261,6 +19344,7 @@ init_cancellation();
 function createScrapeTool(options) {
   const config = options.config || getConfig(options.ctx.cwd);
   const maxScrapeBatches = options.tracker?.getToolLimit("scrape") ?? getMaxScrapeBatches(config);
+  const maxScrapeUrls = getMaxScrapeUrls(config);
   const container = tryGetServiceContainerFromCtx(options.ctx);
   const fallbackState = {
     version: 1,
@@ -19279,7 +19363,7 @@ function createScrapeTool(options) {
     fallbackState.allScrapedLinks = [.../* @__PURE__ */ new Set([...fallbackState.allScrapedLinks, ...links])];
   });
   const ScrapeParamsSchema = Type5.Object({
-    urls: Type5.Array(Type5.String({ description: `The URLs to scrape (only the first ${MAX_SCRAPE_URLS} are fetched per call; extras are reported under "Not Fetched" rather than silently dropped)` }), { minItems: 1, maxItems: 20 }),
+    urls: Type5.Array(Type5.String({ description: `The URLs to scrape (only the first ${maxScrapeUrls} are fetched per call; extras are reported under "Not Fetched" rather than silently dropped)` }), { minItems: 1, maxItems: 20 }),
     maxConcurrency: Type5.Optional(Type5.Number({ default: config.MAX_CONCURRENT_SCRAPES, minimum: 1, maximum: 20 }))
   });
   const trackerLimit = options.tracker?.getToolLimit("scrape");
@@ -19289,7 +19373,7 @@ function createScrapeTool(options) {
     batchProtocolText = `PROTOCOL: Batch 1 \u2192 Batch 2 \u2192 ... (up to ${effectiveLimit} batches)`;
   } else {
     const batchNumbers = Array.from({ length: effectiveLimit }, (_, i) => `Batch ${i + 1}`).join(" \u2192 ");
-    batchProtocolText = `PROTOCOL: ${batchNumbers} (up to ${MAX_SCRAPE_URLS} URLs each).`;
+    batchProtocolText = `PROTOCOL: ${batchNumbers} (up to ${maxScrapeUrls} URLs each).`;
   }
   return {
     name: "scrape",
@@ -19298,7 +19382,7 @@ function createScrapeTool(options) {
     promptSnippet: `Read full content of web pages or PDFs (up to ${maxScrapeBatches} batches)`,
     promptGuidelines: [
       batchProtocolText,
-      `Up to ${MAX_SCRAPE_URLS} URLs per batch.`,
+      `Up to ${maxScrapeUrls} URLs per batch.`,
       "Handshake is ELIMINATED. Start scraping immediately.",
       "PDFs are auto-detected and extracted with high fidelity."
     ],
@@ -19344,8 +19428,8 @@ function createScrapeTool(options) {
         }
       }
       const fetchCandidates = [...dedupedUrls, ...evictedDuplicates];
-      const finalUrls = fetchCandidates.slice(0, MAX_SCRAPE_URLS);
-      const skippedUrls = fetchCandidates.slice(MAX_SCRAPE_URLS);
+      const finalUrls = fetchCandidates.slice(0, maxScrapeUrls);
+      const skippedUrls = fetchCandidates.slice(maxScrapeUrls);
       if (options.getTokensUsed) {
         const ctxWindow = options.contextWindowSize ?? DEFAULT_MODEL_CONTEXT_WINDOW;
         const tokensUsed = options.getTokensUsed();
@@ -19512,7 +19596,7 @@ ${dedupNote}`;
         markdown += `## Not Fetched \u2014 Over Batch Cap (${skippedUrls.length})
 
 `;
-        markdown += `Only the first ${MAX_SCRAPE_URLS} URLs of a batch are fetched. The following URLs were NOT fetched \u2014 no content for them appears above. Request them in a later scrape batch if still needed:
+        markdown += `Only the first ${maxScrapeUrls} URLs of a batch are fetched. The following URLs were NOT fetched \u2014 no content for them appears above. Request them in a later scrape batch if still needed:
 `;
         for (const url of skippedUrls) {
           markdown += `- ${url}
@@ -23685,7 +23769,7 @@ ${effectiveInitialLinks.map((l) => `- ${l}`).join("\n")}`;
   let lastError;
   const researcherExecutionStartMs = Date.now();
   const deliveredSteeringIds = /* @__PURE__ */ new Set();
-  const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "").replace("{{digest_section}}", () => RESEARCHER_DIGEST_SECTION).trim();
+  const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "").replaceAll("{{max_scrape_urls}}", String(config.MAX_SCRAPE_URLS)).replace("{{digest_section}}", () => RESEARCHER_DIGEST_SECTION).trim();
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     if (signal?.aborted) throw new Error("Aborted");
     if (stopRequested()) {
@@ -24215,7 +24299,7 @@ For roughly one in ${this.config.YOUTUBE_QUERY_EVERY_N} of your queries, append
 Your goal is to gather a focused, high-quality pool of initial links.
 
 ## Scrape
-After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to 4 URLs each).
+After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to ${this.config.MAX_SCRAPE_URLS} URLs each).
 Prioritize primary sources and authoritative data.`;
       consumeQueuedMessages(this.options.sessionId);
       const steeringMessages = getSteeringMessages(this.options.sessionId);
@@ -24228,7 +24312,7 @@ Prioritize primary sources and authoritative data.`;
         evidenceLines.push("## Initial Links\nInvestigate these provided URLs first:\n" + this.options.initialLinks.map((l) => `- ${l}`).join("\n"));
       }
       evidenceLines.push(quickEvidenceSection);
-      const prompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "- `search`: Perform broad web searches (Round 1 only).").replace("{{digest_section}}", "").trim();
+      const prompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "- `search`: Perform broad web searches (Round 1 only).").replaceAll("{{max_scrape_urls}}", String(this.config.MAX_SCRAPE_URLS)).replace("{{digest_section}}", "").trim();
       const userMessage = [
         `Goal: ${query}`,
         storeSection.trim(),
@@ -27278,7 +27362,8 @@ async function cmdKnowledgeConfig(kc) {
   scope:   ${kc.mode === "none" ? "disabled here" : kc.mode === "project" ? "this directory only" : "shared across all directories"}
   saved:   per-directory (${cwd})
   db dir:  ${info2.dbDir}
-` + (overridden ? `
+` + (info2.available ? "" : `  state:   ${info2.unavailableReason}
+`) + (overridden ? `
 NOTE: the effective mode here is still '${info2.mode}' because a higher-precedence ${info2.origin} overrides it. Clear that source for this setting to take effect.
 ` : "")
       );
@@ -27293,7 +27378,8 @@ NOTE: the effective mode here is still '${info2.mode}' because a higher-preceden
       `knowledge store mode: ${info.mode}   (source: ${info.origin})
   scope:   ${info.mode === "none" ? "disabled here" : info.mode === "project" ? "this directory only" : "shared across all directories"}
   db dir:  ${info.dbDir}
-
+` + (info.available ? "" : `  state:   ${info.unavailableReason}
+`) + `
 change it for THIS directory:  ${BINARY_NAME} knowledge-config set <none|project|global>
 change the machine-wide default: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> in ${resolvedConfigPaths().configEnv}
 `

package/dist/prompts/researcher.md

@@ -29,10 +29,10 @@ Do not treat any source as inherently objective. Justify its inclusion through t
 Combine historical URLs and search results into a unified pool. Use previous session summaries as a guide for what to expect. Any knowledge-store results in your user message were retrieved automatically for your goal from the **Project Knowledge Store** and/or **User Knowledge Store** — cite them with that store name as their `Source:`.
 
 ### Step 2: Scrape Round 1
-Identify the 4 most promising URLs and scrape them in one batch (the scrape tool accepts up to 6 per call; 4 is your target per round). Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
+Identify the most promising URLs and scrape them in one batch. The scrape tool accepts up to {{max_scrape_urls}} URLs per call — {{max_scrape_urls}} is your target per round, so plan each batch at the full budget rather than stopping early. Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
 
 ### Step 3: Scrape Round 2 (if needed)
-If your first batch did not yield enough material, scrape up to 4 additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
+If your first batch did not yield enough material, scrape up to {{max_scrape_urls}} additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
 - **Complement** your siblings: if they're covering area X, pick your own URLs that go deeper on X or fill adjacent gaps.
 - **Diversify**: if you have sources on an angle no one else is touching, prioritize those to maximize session coverage.
 - **Stay on mission**: regardless of what siblings are doing, make sure you have enough material for your assigned goal.

package/dist/thread-worker.mjs

@@ -2665,8 +2665,10 @@ var ConfigSchema = Type.Object({
   MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1, default: 0.15 }),
   /** Estimated tokens per scrape result for planning (default: 2500) */
   AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 1e4, default: 2500 }),
-  /** Maximum number of concurrent scrapes (default: 3) */
+  /** Maximum concurrent scrapes (default: 3) */
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */

package/docs/ARCHITECTURE.md

@@ -113,7 +113,7 @@ agent to invoke:
 | Tool | Quick | Deep | Backend |
 |------|-------|------|---------|
 | `search` | ✓ | — | DuckDuckGo Lite via the stealth browser |
-| `scrape` | ✓ | ✓ | Batch page fetch → Markdown via the stealth browser (up to 6 URLs per call) |
+| `scrape` | ✓ | ✓ | Batch page fetch → Markdown via the stealth browser (up to MAX_SCRAPE_URLS URLs per call, default 8) |
 | `youtube_transcript` | ✓ | ✓ | YouTube captions via youtubei.js + BotGuard PoToken (≤3 videos by default, configurable 1–5; one call per researcher) |
 | `security_search` | ✓ | ✓ | NVD, CISA KEV, GitHub Advisories, OSV |
 | `stackexchange` | ✓ | ✓ | Stack Exchange network |

package/docs/CHANGELOG.md

@@ -5,6 +5,35 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.10] - 2026-09-02
+
+### Changed
+
+- **`/research-config` now behaves properly on every pi host — TUI, RPC/web-hub GUIs, print/json, and the SDK (`guis` is a family of frontends, not one client).** The interactive settings menu is now gated on `ctx.mode === 'tui'` (not just `ctx.hasUI`, which is true in RPC too): on non-TUI hosts the command replies with a clear non-interactive guidance message instead of silently entering a menu that cannot render. Those hosts get working headless diagnostics instead — `/research-config health` and `/research-config knowledge-status` now run on any host (previously they were reachable only through the menu, even though the tool failure text points users at health). Every destructive `ctx.ui.confirm` now carries a 60s timeout, so no host can wedge on a dialog whose client never responds.
+
+- **The knowledge store's Arrow/LanceDB boundary is now verified in the actual consumer layout on every CI run.** The dev tree resolves a single apache-arrow 21.1.0 (forced by `overrides`), but a fresh npm consumer install resolves two copies — 21.1.0 nested under the package for the direct pin and 18.1.0 hoisted for `@lancedb/lancedb`'s declared peer ceiling. docs/ARCHITECTURE.md pins 21.1.0 for a measured reason and warns mismatched Arrow copies may not interoperate, and this divergence is invisible to the dev-tree tests. The fresh-install CI job now runs `scripts/consumer-interop-probe.cjs`, which replicates `createStoreTable` (same fields, FTS/BTree index set, reopen-from-disk) with the consumer's exact resolution. README gains an Uninstall section documenting that npm 7+ runs no `preuninstall`, so skill links / `~/.pi/research/state` / caches persist unless removed first.
+
+### Fixed
+
+- **Ctrl+C during a research run no longer cancels the run while the editor still holds typed text — it clears the editor first, and only a subsequent Ctrl+C (editor now empty) cancels the research.** The global TUI controller (`src/tui/tui-controller.ts`) treated Ctrl+C and Esc identically as cancel keys, aborting the session's active research runs on every press of either. That fought pi's own keybinding contract: Ctrl+C is `app.clear` ("Clear editor" — pi's `handleCtrlC()` clears the editor text and never interrupts the agent, doubling as the two-press-within-500ms exit), while Esc is `app.interrupt` ("Cancel or abort"), so a user discarding a drafted follow-up mid-run got the whole run killed instead of an empty editor. The controller now reads the editor via `ctx.ui.getEditorText()`: with text present, Ctrl+C falls through untouched (pi clears the text; the run keeps going); with an empty editor it is an explicit cancel gesture and aborts, still scoped to the current pi session as before; Esc keeps its unconditional panic-cancel role; and a throwing `getEditorText()` (ctx gone stale mid-run) is treated as empty — cancel — the safe default. The two gestures coexist with pi's native double-press exit in practice: the clear-then-decide rhythm is naturally slower than the 500ms window, so the human cancel path never trips pi's shutdown. Seven new tests pin the policy (`test/unit/tui/tui-controller.test.ts`).
+
+- **`/research-config` could silently do nothing on RPC hosts.** `ctx.hasUI` is true in RPC, so the old guard passed and `ctx.ui.custom(...)` was entered even though no non-TUI host can render or answer the menu. The mode gate in the change above makes the menu reachable only on a real TUI and hands every other host the guidance message plus the headless `health` / `knowledge-status` diagnostics.
+
+### Verified
+
+- 3,027 unit tests over 240 files (14 new across this Unreleased cycle: 7 Ctrl+C-policy + 7 host-mode), ESLint clean on the touched source, all four type-checks (TS 6 src + tests, TS 7 native src + tests), and dependency-cruiser clean (238 modules / 1,218 dependencies).
+
+
+## [1.6.9] - 2026-09-01
+
+### Changed
+
+- **`MAX_SCRAPE_URLS` is now a real config setting, raised to 8 (was a hardcoded constant capped at 6).** The per-batch page cap joined every other scrape knob in the Config schema (`PI_RESEARCH_MAX_SCRAPE_URLS`, range 1–20, default 8): env-var parsing, the config.env bridge, persistence, and the bundled thread-worker schema mirror all follow, and `src/tools/scrape.ts` resolves one `getMaxScrapeUrls(config)` value that feeds the tool-schema description, the batch-protocol text, the per-call slice, and the "Not Fetched — Over Batch Cap" report, so the enforced cap and every string the model sees are the same number by construction (previously the constant was 6 while the researcher prompt separately steered a lower target). `src/constants.ts` keeps `MAX_SCRAPE_URLS` as a documented schema-default mirror plus the accessor, mirroring `MAX_GATHERING_CALLS`. The researcher prompt no longer steers below the cap: a new {{max_scrape_urls}} placeholder — substituted from config in BOTH call sites (deep researcher-executor and the quick orchestrator, each of which maintains its own replace chain) — states the per-call maximum as the batch target ("{{max_scrape_urls}} is your target per round, so plan each batch at the full budget"), so the prompt can never drift from the enforced cap the way the old hardcoded targets (4, then 6) already had from the constants-derived tool text. The shipped-render tests for both paths pin the substitution. Context-gate note: batches of 8 bill 8 × `AVG_TOKENS_PER_SCRAPE` against the gate fraction, so installs pinning a high per-URL estimate with a small window (the code comment records an 8-URL batch blocked on a 128k model) may see more `context_limit` blocks — tune `PI_RESEARCH_AVG_TOKENS_PER_SCRAPE` if so. `MAX_CONCURRENT_SCRAPES` (fetch parallelism) is untouched at its long-standing default of 3.
+
+### Fixed
+
+- **A knowledge store whose packages are not even resolvable is now cleanly OFF — at the settings layer, not buried in runtime degrade.** Since 1.6.7 the store correctly memoized DISABLED when `@huggingface/transformers` (optional dep) was skipped at install or the `@lancedb/lancedb` install was broken — but only AFTER running the full init: `MAX_INIT_RETRIES` (5) attempts of exponential backoff (~31s) per fresh process, spent discovering what `require.resolve` already knows, and every settings surface kept advertising a `KNOWLEDGE_STORE_MODE` that could never run. Now a new execution-free probe (`src/knowledge/availability.ts` — `require.resolve` only, never loads the native modules) is consulted BEFORE the retry loop: `createKnowledgeStoreComponents` returns null immediately (the same clean OFF verdict mode=`none` produces), and `KnowledgeStoreService` records it as DISABLED(`native`) — permanent for the process, never revived by a mode flip, mirroring the existing catch-path semantics. The settings surfaces now tell the truth about it everywhere: `describeKnowledgeStoreMode` gained `available`/`unavailableReason` (flows into `knowledge-config --json` automatically), the `knowledge-config` text output and the `/research-config` TUI's Knowledge Mode item say what is missing and how to repair it, and the healthcheck's 'disabled' verdict names the actual missing packages instead of the generic platform line. One more surface joined them: the `research_knowledge_search` miss message no longer tells a package-gap host the store is "disabled in settings" — a 'native' service disable now reports `store_packages_missing` (the repair is installing the package, not flipping a Knowledge Mode toggle that cannot help), while a true mode disable keeps `store_disabled` and an in-flight/failed init keeps `store_not_ready`. Unit tests pin the probe, the fail-fast (embedder factory never invoked), the native-vs-mode disable distinction, and the miss-reason split.
+
 ## [1.6.8] - 2026-08-31
 
 ### Changed

package/docs/CONFIGURATION.md

@@ -71,6 +71,7 @@ Research
 | `PI_RESEARCH_MAX_SCRAPE_BATCHES` (TUI) | `2` | 0–99 | Scrape batches per researcher (0 = unlimited). When prompt caching is known to be active for the resolved research model (Anthropic API models, or a provider route explicitly configured for Anthropic-style cache control), the effective limit is this value plus one — the cached prompt prefix makes the extra batch cheap. |
 | `PI_RESEARCH_MAX_GATHERING_CALLS` | `12` | 1–100 | Shared web-gathering calls per researcher (`search` + `security_search` + `stackexchange` + `youtube_transcript`). |
 | `PI_RESEARCH_MAX_CONCURRENT_SCRAPES` | `3` | 1–20 | Concurrent URLs fetched per scrape batch. |
+| `PI_RESEARCH_MAX_SCRAPE_URLS` | `8` | 1–20 | Maximum URLs fetched per scrape batch. URLs past the cap are listed under "Not Fetched — Over Batch Cap" and must be requested in a later batch. (Promoted from a hardcoded constant so it is env/config-file tunable like every other scrape knob.) |
 | `PI_RESEARCH_MAX_RETRIES` | `2` | 0–5 | Retries per researcher request. |
 | `PI_RESEARCH_RETRY_DELAY_MS` | `2000` | 100–10000 | Base delay between retries. |
 | `PI_RESEARCH_MAX_FAILED_RESEARCHERS` | `2` | 1–10 | Unique researcher failures that abort the whole run. Raise to let slower, still-in-flight researchers finish before giving up. |
@@ -121,7 +122,7 @@ See the [knowledge store doc](KNOWLEDGE-STORE.md) for what each value does.
 
 | Variable | Default | Range | Description |
 |----------|---------|-------|-------------|
-| `PI_RESEARCH_KNOWLEDGE_STORE_MODE` (TUI) `[project]` | `global` | none · project · global | Store scope: one shared store across every directory (`global`), scoped to the current directory (`project`), or disabled (`none`). |
+| `PI_RESEARCH_KNOWLEDGE_STORE_MODE` (TUI) `[project]` | `global` | none · project · global | Store scope: one shared store across every directory (`global`), scoped to the current directory (`project`), or disabled (`none`). Independently of this setting, the store is cleanly OFF when its required packages are not installed (optional `@huggingface/transformers` skipped at install, broken `@lancedb/lancedb`): init fails fast instead of retry-storming, and `pi-research knowledge-config`, the `/research-config` menu, and the healthcheck all name the missing package and the repair. |
 | `PI_RESEARCH_EMBEDDING_MODEL` (TUI) | `onnx-community/granite-embedding-small-english-r2-ONNX` | — | Embedding model. Changing it clears the store and starts fresh. |
 | `PI_RESEARCH_EMBEDDING_DEVICE` (TUI) | `auto` | auto · webgpu · cpu | Inference backend. `auto` probes WebGPU viability out-of-process and falls back to CPU; `cpu` forces CPU; `webgpu` forces the GPU path with no probe (advanced — can hard-crash on a software GPU). The TUI exposes only `auto` (as "GPU") and `cpu`. |
 | `PI_RESEARCH_CACHE_TTL_DAYS` (TUI) | `30` | 1–365 | How long cached findings are kept before eviction. |

package/docs/KNOWLEDGE-STORE.md

@@ -144,17 +144,25 @@ platform/architecture pairs:
 | Windows | x64 / arm64 | Supported |
 
 Intel Macs (`darwin-x64`) have no prebuilt binary for either component, so the
-knowledge store cannot run there. The degradation is automatic:
+knowledge store cannot run there. The same clean OFF applies on any platform when a
+required package is not installed at all — optional `@huggingface/transformers` is
+skipped by `npm install --omit=optional` (and by an npm optional-dependency bug), or
+the `@lancedb/lancedb` install is broken. The degradation is automatic and fast:
 
 - Research still works. Search, scraping, YouTube transcripts, the security databases,
   Stack Exchange, planning, and synthesis are unaffected — only the store is missing.
-- The health check reports the store as disabled, not unhealthy: its health component
-  returns disabled (native embedding/vector stack unavailable on this platform) with a
-  healthy status, so the missing component does not drag overall health to "unhealthy"
-  or block a quick (depth-0) run.
-- `research_knowledge_search` does no caching or retrieval; every run goes to the live web.
-
-No configuration is needed on an Intel Mac.
+- The store fails fast. Missing packages are detected by resolution before any
+  initialization is attempted, so there is no retry storm — the store simply starts OFF.
+- The settings surfaces say why. `pi-research knowledge-config`, the `/research-config`
+  menu, and the health check name the missing package and the repair (install the
+  optional dependencies) instead of advertising a mode the store cannot honor. A
+  `research_knowledge_search` miss reports the package gap rather than pointing at a
+  settings toggle that cannot help.
+- The health check reports the store as disabled, not unhealthy, so the missing
+  component does not drag overall health to "unhealthy" or block a quick (depth-0) run.
+
+No configuration is needed: install the package (a full install including optional
+dependencies) and the store comes up; a mode flip alone cannot revive it mid-process.
 
 ### Retention and eviction
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.6.8",
+  "version": "1.6.10",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,

package/src/cli.ts

@@ -916,6 +916,7 @@ export async function cmdKnowledgeConfig(kc: NonNullable<ParsedArgs['knowledgeCo
         `  scope:   ${kc.mode === 'none' ? 'disabled here' : kc.mode === 'project' ? 'this directory only' : 'shared across all directories'}\n` +
         `  saved:   per-directory (${cwd})\n` +
         `  db dir:  ${info.dbDir}\n` +
+        (info.available ? '' : `  state:   ${info.unavailableReason}\n`) +
         (overridden
           ? `\nNOTE: the effective mode here is still '${info.mode}' because a higher-precedence ${info.origin} overrides it. Clear that source for this setting to take effect.\n`
           : ''),
@@ -933,6 +934,7 @@ export async function cmdKnowledgeConfig(kc: NonNullable<ParsedArgs['knowledgeCo
       `knowledge store mode: ${info.mode}   (source: ${info.origin})\n` +
       `  scope:   ${info.mode === 'none' ? 'disabled here' : info.mode === 'project' ? 'this directory only' : 'shared across all directories'}\n` +
       `  db dir:  ${info.dbDir}\n` +
+      (info.available ? '' : `  state:   ${info.unavailableReason}\n`) +
       `\nchange it for THIS directory:  ${BINARY_NAME} knowledge-config set <none|project|global>\n` +
       `change the machine-wide default: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> in ${resolvedConfigPaths().configEnv}\n`,
     );