@lincoln504/pi-research 1.6.8 → 1.6.9

package/agent-skill/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.6.8", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.6.9", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---
 

package/dist/cli.mjs

@@ -2387,6 +2387,46 @@ var init_atomic_replace = __esm({
   }
 });
 
+// src/knowledge/availability.ts
+import { createRequire as createRequire3 } from "node:module";
+function probeKnowledgeStoreAvailability(resolver = defaultResolver) {
+  if (cachedIsValidFor(resolver)) return _cached;
+  const missing = [];
+  for (const pkg of REQUIRED_PACKAGES) {
+    try {
+      resolver(pkg);
+    } catch {
+      missing.push(pkg);
+    }
+  }
+  const result = { available: missing.length === 0, missing };
+  if (resolver === defaultResolver) _cached = result;
+  return result;
+}
+function cachedIsValidFor(resolver) {
+  return _cached !== null && resolver === defaultResolver;
+}
+function describeKnowledgeStoreUnavailability(availability) {
+  if (availability.available) return "";
+  const parts = availability.missing.map(
+    (pkg) => pkg === "@huggingface/transformers" ? `'${pkg}' (optional embedding dependency \u2014 likely skipped at install time)` : `'${pkg}' (broken install \u2014 native vector store missing)`
+  );
+  return `knowledge store is OFF: required package${parts.length > 1 ? "s" : ""} not resolvable \u2014 ${parts.join("; ")}. Repair with a full install of optional dependencies (e.g. 'npm install' including optional deps, or pi install npm:@lincoln504/pi-research without --omit=optional).`;
+}
+var REQUIRED_PACKAGES, _cached, defaultResolver;
+var init_availability = __esm({
+  "src/knowledge/availability.ts"() {
+    "use strict";
+    REQUIRED_PACKAGES = ["@lancedb/lancedb", "@huggingface/transformers"];
+    _cached = null;
+    defaultResolver = (specifier) => (
+      // import.meta.url survives esbuild's ESM output; createRequire only builds a
+      // resolver — it never evaluates the specifier.
+      createRequire3(import.meta.url).resolve(specifier)
+    );
+  }
+});
+
 // src/config.ts
 import * as fs5 from "node:fs";
 import * as path5 from "node:path";
@@ -2593,7 +2633,19 @@ function describeKnowledgeStoreMode(cwd = process.cwd(), iface) {
     else if (iface && fileHasKey(getInterfaceEnvFilePath(iface))) origin = `${iface}.env overlay`;
     else if (fileHasKey(getGlobalEnvFilePath())) origin = "config.env (machine-wide default)";
   }
-  return { mode: cfg.KNOWLEDGE_STORE_MODE, origin, dbDir: getDbDir(cfg, cwd) };
+  return {
+    mode: cfg.KNOWLEDGE_STORE_MODE,
+    origin,
+    dbDir: getDbDir(cfg, cwd),
+    ...availabilityFields()
+  };
+}
+function availabilityFields() {
+  const availability = probeKnowledgeStoreAvailability();
+  return {
+    available: availability.available,
+    unavailableReason: describeKnowledgeStoreUnavailability(availability)
+  };
 }
 function parseDotEnv(content) {
   const out = {};
@@ -2744,6 +2796,7 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
     PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: String(config.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING),
     PI_RESEARCH_AVG_TOKENS_PER_SCRAPE: String(config.AVG_TOKENS_PER_SCRAPE),
     PI_RESEARCH_MAX_CONCURRENT_SCRAPES: String(config.MAX_CONCURRENT_SCRAPES),
+    PI_RESEARCH_MAX_SCRAPE_URLS: String(config.MAX_SCRAPE_URLS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS: String(config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS: String(config.YOUTUBE_TRANSCRIPT_TIMEOUT_MS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG: config.YOUTUBE_TRANSCRIPT_LANG,
@@ -2905,6 +2958,7 @@ function createConfig(env, processEnv) {
     MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: parseEnvNumber(e, "PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING", DEFAULTS.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING, 0.05, 1),
     AVG_TOKENS_PER_SCRAPE: parseEnvNumber(e, "PI_RESEARCH_AVG_TOKENS_PER_SCRAPE", DEFAULTS.AVG_TOKENS_PER_SCRAPE, 500, 1e4),
     MAX_CONCURRENT_SCRAPES: parseEnvNumber(e, "PI_RESEARCH_MAX_CONCURRENT_SCRAPES", DEFAULTS.MAX_CONCURRENT_SCRAPES, 1, 20, true),
+    MAX_SCRAPE_URLS: parseEnvNumber(e, "PI_RESEARCH_MAX_SCRAPE_URLS", DEFAULTS.MAX_SCRAPE_URLS, 1, 20, true),
     YOUTUBE_TRANSCRIPT_MAX_VIDEOS: parseEnvNumber(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS", DEFAULTS.YOUTUBE_TRANSCRIPT_MAX_VIDEOS, 1, 5, true),
     YOUTUBE_TRANSCRIPT_TIMEOUT_MS: parseEnvNumber(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS", DEFAULTS.YOUTUBE_TRANSCRIPT_TIMEOUT_MS, 5e3, 12e4),
     YOUTUBE_TRANSCRIPT_LANG: parseEnvString(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG", DEFAULTS.YOUTUBE_TRANSCRIPT_LANG),
@@ -3002,6 +3056,7 @@ var init_config = __esm({
     init_host_config();
     init_text_utils();
     init_atomic_replace();
+    init_availability();
     ConfigSchema = Type.Object({
       /** Per-researcher timeout in milliseconds (default: 300000, range: 3-30 min) */
       RESEARCHER_TIMEOUT_MS: Type.Number({ minimum: 18e4, maximum: 18e5, default: 3e5 }),
@@ -3046,8 +3101,10 @@ var init_config = __esm({
       MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1, default: 0.15 }),
       /** Estimated tokens per scrape result for planning (default: 2500) */
       AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 1e4, default: 2500 }),
-      /** Maximum number of concurrent scrapes (default: 3) */
+      /** Maximum concurrent scrapes (default: 3) */
       MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+      /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+      MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
       /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
       YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
       /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -3184,6 +3241,7 @@ var init_config = __esm({
       "PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING",
       "PI_RESEARCH_AVG_TOKENS_PER_SCRAPE",
       "PI_RESEARCH_MAX_CONCURRENT_SCRAPES",
+      "PI_RESEARCH_MAX_SCRAPE_URLS",
       "PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS",
       "PI_RESEARCH_SEARCH_TIMEOUT_MS",
       "PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS",
@@ -3240,6 +3298,13 @@ function getMaxScrapeBatches(config, cachingActive = false) {
     return 2;
   }
 }
+function getMaxScrapeUrls(config) {
+  try {
+    return (config || getConfig()).MAX_SCRAPE_URLS;
+  } catch {
+    return MAX_SCRAPE_URLS;
+  }
+}
 function resolveExcludedTools(callerExclusions, configDisabled = []) {
   return [.../* @__PURE__ */ new Set([...DEFAULT_EXCLUDED_TOOLS, ...callerExclusions ?? [], ...configDisabled])];
 }
@@ -3250,7 +3315,7 @@ var init_constants = __esm({
     init_config();
     MAX_GATHERING_CALLS = 12;
     MAX_GREP_CALLS = 30;
-    MAX_SCRAPE_URLS = 6;
+    MAX_SCRAPE_URLS = 8;
     MAX_TEAM_SIZE_LEVEL_1 = 2;
     MAX_TEAM_SIZE_LEVEL_2 = 3;
     MAX_TEAM_SIZE_LEVEL_3 = 5;
@@ -9007,11 +9072,15 @@ function getMigrationStrategy(config) {
   logger.warn(`[knowledge] Invalid migration strategy '${strategy}'. Valid options: drop, re-embed, backup. Falling back to default (backup).`);
   return void 0;
 }
-async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory, withLock, config = getConfig(), workspace = process.cwd()) {
+async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory, withLock, config = getConfig(), workspace = process.cwd(), availability = probeKnowledgeStoreAvailability()) {
   if (config.KNOWLEDGE_STORE_MODE === "none") {
     logger.debug("[knowledge] Knowledge store is disabled in configuration");
     return null;
   }
+  if (!availability.available) {
+    logger.warn(`[knowledge] ${describeKnowledgeStoreUnavailability(availability)}`);
+    return null;
+  }
   validateConfig(config);
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {
     let store;
@@ -9089,6 +9158,7 @@ var init_knowledge = __esm({
     init_config();
     init_logger();
     init_model_config();
+    init_availability();
     init_model_config();
     MAX_INIT_RETRIES = 5;
     BASE_RETRY_DELAY_MS = 1e3;
@@ -9115,6 +9185,7 @@ var init_knowledge_store_service = __esm({
     init_embedding_factory();
     init_config();
     init_embedder_utils();
+    init_availability();
     KnowledgeStoreService = class {
       name = ServiceNames.KNOWLEDGE_STORE;
       lifecycle = "uninitialized" /* UNINITIALIZED */;
@@ -9255,9 +9326,10 @@ var init_knowledge_store_service = __esm({
               }
             }
             if (!components) {
-              logger.debug("[KnowledgeStoreService] Knowledge store is disabled. Setting lifecycle to DISABLED.");
+              const depsMissing = !probeKnowledgeStoreAvailability().available;
+              logger.debug(`[KnowledgeStoreService] Knowledge store is disabled (${depsMissing ? "dependencies missing" : "mode = none"}). Setting lifecycle to DISABLED.`);
               this.lifecycle = "disabled" /* DISABLED */;
-              this._disabledReason = "mode";
+              this._disabledReason = depsMissing ? "native" : "mode";
               this._embedder = null;
               this._store = null;
               this._writerQueue = null;
@@ -10104,6 +10176,9 @@ function missResult(reason) {
     case "store_disabled":
       text = "No results found (knowledge store is disabled in settings). Live research can get the info.";
       break;
+    case "store_packages_missing":
+      text = "No results found (knowledge store is off \u2014 its required packages are not installed on this host, so it stays off regardless of settings). Live research can get the info.";
+      break;
     case "store_not_ready":
       text = "No results found (knowledge store is initializing). Live research can get the info.";
       break;
@@ -10166,8 +10241,10 @@ function createResearchKnowledgeSearchTool(iface) {
         }
         const store = await storeService.getStore();
         if (!store) {
-          const lifecycle = storeService.lifecycle;
-          return missResult(lifecycle === "disabled" ? "store_disabled" : "store_not_ready");
+          const disabledReason = storeService.getDisabledReason();
+          if (disabledReason === "native") return missResult("store_packages_missing");
+          if (disabledReason === "mode") return missResult("store_disabled");
+          return missResult("store_not_ready");
         }
         const count = await store.count();
         if (count === 0) {
@@ -12537,7 +12614,7 @@ import * as crypto4 from "node:crypto";
 import { join as join8 } from "node:path";
 import { existsSync as existsSync6, mkdirSync as mkdirSync3 } from "node:fs";
 import { platform, homedir as homedir3 } from "node:os";
-import { createRequire as createRequire3 } from "node:module";
+import { createRequire as createRequire4 } from "node:module";
 function getWindowsCamoufoxDir() {
   return join8(homedir3(), "AppData", "Local", "camoufox", "camoufox", "Cache");
 }
@@ -12655,7 +12732,7 @@ function probeBrowserNativeDeps(load = defaultNativeDepLoad) {
   }
 }
 function defaultNativeDepLoad() {
-  const require2 = createRequire3(import.meta.url);
+  const require2 = createRequire4(import.meta.url);
   const Database = require2("better-sqlite3");
   const db = new Database(":memory:");
   db.close();
@@ -15988,7 +16065,7 @@ import { FixedClusterPool, WorkerChoiceStrategies } from "poolifier";
 // src/infrastructure/browser/ensure-browser.ts
 init_logger();
 import { spawn as spawn2 } from "node:child_process";
-import { createRequire as createRequire4 } from "node:module";
+import { createRequire as createRequire5 } from "node:module";
 import { createHash as createHash2 } from "node:crypto";
 import { existsSync as existsSync7, readdirSync as readdirSync3, statSync as statSync3, fstatSync, openSync as openSync4, closeSync as closeSync4, rmSync } from "node:fs";
 import { join as join11, dirname as dirname8 } from "node:path";
@@ -16015,7 +16092,7 @@ function isBrowserBinaryPresent() {
 }
 function resolveCamoufoxBin() {
   try {
-    const require2 = createRequire4(import.meta.url);
+    const require2 = createRequire5(import.meta.url);
     const pkgJson = require2.resolve("camoufox-js/package.json");
     const bin = join11(dirname8(pkgJson), "..", ".bin", process.platform === "win32" ? "camoufox-js.cmd" : "camoufox-js");
     if (existsSync7(bin)) return bin;
@@ -17939,6 +18016,7 @@ healthRegistry.initialize().catch(() => {
 
 // src/healthcheck/index.ts
 init_embedder_utils();
+init_availability();
 async function checkBrowserCapability() {
   const mockMode = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" && process.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
   if (isBrowserAvailable() || mockMode) {
@@ -17957,6 +18035,11 @@ async function checkBrowserCapability() {
     return { healthy: false, error: 'Camoufox (browser) not found. Run "npx camoufox-js fetch" to install the browser.' };
   }
 }
+function knowledgeStoreDisabledStatus() {
+  const availability = probeKnowledgeStoreAvailability();
+  const detail = availability.available ? "native embedding/vector stack unavailable on this platform" : `required packages not resolvable \u2014 ${availability.missing.join(", ")}`;
+  return `disabled (${detail})`;
+}
 function registerHealthChecks(registry, container = getServiceContainer()) {
   const config = getConfig();
   const healthTimeoutMs = config.HEALTH_CHECK_TIMEOUT_MS;
@@ -17988,7 +18071,7 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       if (existing?.lifecycle === "disabled" /* DISABLED */) {
         const reason = existing.getDisabledReason();
         return { healthy: true, diagnostic: {
-          status: reason === "native" ? "disabled (native embedding/vector stack unavailable on this platform)" : "disabled"
+          status: reason === "native" ? knowledgeStoreDisabledStatus() : "disabled"
         } };
       }
       const status = existing?.lifecycle === "initializing" /* INITIALIZING */ ? "initializing" : existing?.lifecycle === "disposed" /* DISPOSED */ || existing?.lifecycle === "disposing" /* DISPOSING */ ? "disposed (not running)" : "ready (idle)";
@@ -18007,12 +18090,12 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       return { healthy: true, diagnostic: { status: "disabled in config" } };
     }
     if (isNativeStackUnavailableError(initError)) {
-      return { healthy: true, diagnostic: { status: "disabled (native embedding/vector stack unavailable on this platform)" } };
+      return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
     }
     if (service?.lifecycle === "disabled" /* DISABLED */) {
       const reason = service.getDisabledReason();
       return { healthy: true, diagnostic: {
-        status: reason === "native" ? "disabled (native embedding/vector stack unavailable on this platform)" : "disabled"
+        status: reason === "native" ? knowledgeStoreDisabledStatus() : "disabled"
       } };
     }
     try {
@@ -18061,7 +18144,7 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       };
     } catch (e) {
       if (isNativeStackUnavailableError(e)) {
-        return { healthy: true, diagnostic: { status: "disabled (native embedding/vector stack unavailable on this platform)" } };
+        return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
       }
       return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
     }
@@ -19261,6 +19344,7 @@ init_cancellation();
 function createScrapeTool(options) {
   const config = options.config || getConfig(options.ctx.cwd);
   const maxScrapeBatches = options.tracker?.getToolLimit("scrape") ?? getMaxScrapeBatches(config);
+  const maxScrapeUrls = getMaxScrapeUrls(config);
   const container = tryGetServiceContainerFromCtx(options.ctx);
   const fallbackState = {
     version: 1,
@@ -19279,7 +19363,7 @@ function createScrapeTool(options) {
     fallbackState.allScrapedLinks = [.../* @__PURE__ */ new Set([...fallbackState.allScrapedLinks, ...links])];
   });
   const ScrapeParamsSchema = Type5.Object({
-    urls: Type5.Array(Type5.String({ description: `The URLs to scrape (only the first ${MAX_SCRAPE_URLS} are fetched per call; extras are reported under "Not Fetched" rather than silently dropped)` }), { minItems: 1, maxItems: 20 }),
+    urls: Type5.Array(Type5.String({ description: `The URLs to scrape (only the first ${maxScrapeUrls} are fetched per call; extras are reported under "Not Fetched" rather than silently dropped)` }), { minItems: 1, maxItems: 20 }),
     maxConcurrency: Type5.Optional(Type5.Number({ default: config.MAX_CONCURRENT_SCRAPES, minimum: 1, maximum: 20 }))
   });
   const trackerLimit = options.tracker?.getToolLimit("scrape");
@@ -19289,7 +19373,7 @@ function createScrapeTool(options) {
     batchProtocolText = `PROTOCOL: Batch 1 \u2192 Batch 2 \u2192 ... (up to ${effectiveLimit} batches)`;
   } else {
     const batchNumbers = Array.from({ length: effectiveLimit }, (_, i) => `Batch ${i + 1}`).join(" \u2192 ");
-    batchProtocolText = `PROTOCOL: ${batchNumbers} (up to ${MAX_SCRAPE_URLS} URLs each).`;
+    batchProtocolText = `PROTOCOL: ${batchNumbers} (up to ${maxScrapeUrls} URLs each).`;
   }
   return {
     name: "scrape",
@@ -19298,7 +19382,7 @@ function createScrapeTool(options) {
     promptSnippet: `Read full content of web pages or PDFs (up to ${maxScrapeBatches} batches)`,
     promptGuidelines: [
       batchProtocolText,
-      `Up to ${MAX_SCRAPE_URLS} URLs per batch.`,
+      `Up to ${maxScrapeUrls} URLs per batch.`,
       "Handshake is ELIMINATED. Start scraping immediately.",
       "PDFs are auto-detected and extracted with high fidelity."
     ],
@@ -19344,8 +19428,8 @@ function createScrapeTool(options) {
         }
       }
       const fetchCandidates = [...dedupedUrls, ...evictedDuplicates];
-      const finalUrls = fetchCandidates.slice(0, MAX_SCRAPE_URLS);
-      const skippedUrls = fetchCandidates.slice(MAX_SCRAPE_URLS);
+      const finalUrls = fetchCandidates.slice(0, maxScrapeUrls);
+      const skippedUrls = fetchCandidates.slice(maxScrapeUrls);
       if (options.getTokensUsed) {
         const ctxWindow = options.contextWindowSize ?? DEFAULT_MODEL_CONTEXT_WINDOW;
         const tokensUsed = options.getTokensUsed();
@@ -19512,7 +19596,7 @@ ${dedupNote}`;
         markdown += `## Not Fetched \u2014 Over Batch Cap (${skippedUrls.length})
 
 `;
-        markdown += `Only the first ${MAX_SCRAPE_URLS} URLs of a batch are fetched. The following URLs were NOT fetched \u2014 no content for them appears above. Request them in a later scrape batch if still needed:
+        markdown += `Only the first ${maxScrapeUrls} URLs of a batch are fetched. The following URLs were NOT fetched \u2014 no content for them appears above. Request them in a later scrape batch if still needed:
 `;
         for (const url of skippedUrls) {
           markdown += `- ${url}
@@ -23685,7 +23769,7 @@ ${effectiveInitialLinks.map((l) => `- ${l}`).join("\n")}`;
   let lastError;
   const researcherExecutionStartMs = Date.now();
   const deliveredSteeringIds = /* @__PURE__ */ new Set();
-  const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "").replace("{{digest_section}}", () => RESEARCHER_DIGEST_SECTION).trim();
+  const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "").replaceAll("{{max_scrape_urls}}", String(config.MAX_SCRAPE_URLS)).replace("{{digest_section}}", () => RESEARCHER_DIGEST_SECTION).trim();
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     if (signal?.aborted) throw new Error("Aborted");
     if (stopRequested()) {
@@ -24215,7 +24299,7 @@ For roughly one in ${this.config.YOUTUBE_QUERY_EVERY_N} of your queries, append
 Your goal is to gather a focused, high-quality pool of initial links.
 
 ## Scrape
-After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to 4 URLs each).
+After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to ${this.config.MAX_SCRAPE_URLS} URLs each).
 Prioritize primary sources and authoritative data.`;
       consumeQueuedMessages(this.options.sessionId);
       const steeringMessages = getSteeringMessages(this.options.sessionId);
@@ -24228,7 +24312,7 @@ Prioritize primary sources and authoritative data.`;
         evidenceLines.push("## Initial Links\nInvestigate these provided URLs first:\n" + this.options.initialLinks.map((l) => `- ${l}`).join("\n"));
       }
       evidenceLines.push(quickEvidenceSection);
-      const prompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "- `search`: Perform broad web searches (Round 1 only).").replace("{{digest_section}}", "").trim();
+      const prompt = injectCurrentDate(researcherPromptTemplate, "researcher").replace("{{extra_tool_guidelines}}", "- `search`: Perform broad web searches (Round 1 only).").replaceAll("{{max_scrape_urls}}", String(this.config.MAX_SCRAPE_URLS)).replace("{{digest_section}}", "").trim();
       const userMessage = [
         `Goal: ${query}`,
         storeSection.trim(),
@@ -27278,7 +27362,8 @@ async function cmdKnowledgeConfig(kc) {
   scope:   ${kc.mode === "none" ? "disabled here" : kc.mode === "project" ? "this directory only" : "shared across all directories"}
   saved:   per-directory (${cwd})
   db dir:  ${info2.dbDir}
-` + (overridden ? `
+` + (info2.available ? "" : `  state:   ${info2.unavailableReason}
+`) + (overridden ? `
 NOTE: the effective mode here is still '${info2.mode}' because a higher-precedence ${info2.origin} overrides it. Clear that source for this setting to take effect.
 ` : "")
       );
@@ -27293,7 +27378,8 @@ NOTE: the effective mode here is still '${info2.mode}' because a higher-preceden
       `knowledge store mode: ${info.mode}   (source: ${info.origin})
   scope:   ${info.mode === "none" ? "disabled here" : info.mode === "project" ? "this directory only" : "shared across all directories"}
   db dir:  ${info.dbDir}
-
+` + (info.available ? "" : `  state:   ${info.unavailableReason}
+`) + `
 change it for THIS directory:  ${BINARY_NAME} knowledge-config set <none|project|global>
 change the machine-wide default: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> in ${resolvedConfigPaths().configEnv}
 `

package/dist/prompts/researcher.md

@@ -29,10 +29,10 @@ Do not treat any source as inherently objective. Justify its inclusion through t
 Combine historical URLs and search results into a unified pool. Use previous session summaries as a guide for what to expect. Any knowledge-store results in your user message were retrieved automatically for your goal from the **Project Knowledge Store** and/or **User Knowledge Store** — cite them with that store name as their `Source:`.
 
 ### Step 2: Scrape Round 1
-Identify the 4 most promising URLs and scrape them in one batch (the scrape tool accepts up to 6 per call; 4 is your target per round). Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
+Identify the most promising URLs and scrape them in one batch. The scrape tool accepts up to {{max_scrape_urls}} URLs per call — {{max_scrape_urls}} is your target per round, so plan each batch at the full budget rather than stopping early. Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
 
 ### Step 3: Scrape Round 2 (if needed)
-If your first batch did not yield enough material, scrape up to 4 additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
+If your first batch did not yield enough material, scrape up to {{max_scrape_urls}} additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
 - **Complement** your siblings: if they're covering area X, pick your own URLs that go deeper on X or fill adjacent gaps.
 - **Diversify**: if you have sources on an angle no one else is touching, prioritize those to maximize session coverage.
 - **Stay on mission**: regardless of what siblings are doing, make sure you have enough material for your assigned goal.

package/dist/thread-worker.mjs

@@ -2665,8 +2665,10 @@ var ConfigSchema = Type.Object({
   MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1, default: 0.15 }),
   /** Estimated tokens per scrape result for planning (default: 2500) */
   AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 1e4, default: 2500 }),
-  /** Maximum number of concurrent scrapes (default: 3) */
+  /** Maximum concurrent scrapes (default: 3) */
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */

package/docs/ARCHITECTURE.md

@@ -113,7 +113,7 @@ agent to invoke:
 | Tool | Quick | Deep | Backend |
 |------|-------|------|---------|
 | `search` | ✓ | — | DuckDuckGo Lite via the stealth browser |
-| `scrape` | ✓ | ✓ | Batch page fetch → Markdown via the stealth browser (up to 6 URLs per call) |
+| `scrape` | ✓ | ✓ | Batch page fetch → Markdown via the stealth browser (up to MAX_SCRAPE_URLS URLs per call, default 8) |
 | `youtube_transcript` | ✓ | ✓ | YouTube captions via youtubei.js + BotGuard PoToken (≤3 videos by default, configurable 1–5; one call per researcher) |
 | `security_search` | ✓ | ✓ | NVD, CISA KEV, GitHub Advisories, OSV |
 | `stackexchange` | ✓ | ✓ | Stack Exchange network |

package/docs/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.9] - 2026-09-01
+
+### Changed
+
+- **`MAX_SCRAPE_URLS` is now a real config setting, raised to 8 (was a hardcoded constant capped at 6).** The per-batch page cap joined every other scrape knob in the Config schema (`PI_RESEARCH_MAX_SCRAPE_URLS`, range 1–20, default 8): env-var parsing, the config.env bridge, persistence, and the bundled thread-worker schema mirror all follow, and `src/tools/scrape.ts` resolves one `getMaxScrapeUrls(config)` value that feeds the tool-schema description, the batch-protocol text, the per-call slice, and the "Not Fetched — Over Batch Cap" report, so the enforced cap and every string the model sees are the same number by construction (previously the constant was 6 while the researcher prompt separately steered a lower target). `src/constants.ts` keeps `MAX_SCRAPE_URLS` as a documented schema-default mirror plus the accessor, mirroring `MAX_GATHERING_CALLS`. The researcher prompt no longer steers below the cap: a new {{max_scrape_urls}} placeholder — substituted from config in BOTH call sites (deep researcher-executor and the quick orchestrator, each of which maintains its own replace chain) — states the per-call maximum as the batch target ("{{max_scrape_urls}} is your target per round, so plan each batch at the full budget"), so the prompt can never drift from the enforced cap the way the old hardcoded targets (4, then 6) already had from the constants-derived tool text. The shipped-render tests for both paths pin the substitution. Context-gate note: batches of 8 bill 8 × `AVG_TOKENS_PER_SCRAPE` against the gate fraction, so installs pinning a high per-URL estimate with a small window (the code comment records an 8-URL batch blocked on a 128k model) may see more `context_limit` blocks — tune `PI_RESEARCH_AVG_TOKENS_PER_SCRAPE` if so. `MAX_CONCURRENT_SCRAPES` (fetch parallelism) is untouched at its long-standing default of 3.
+
+### Fixed
+
+- **A knowledge store whose packages are not even resolvable is now cleanly OFF — at the settings layer, not buried in runtime degrade.** Since 1.6.7 the store correctly memoized DISABLED when `@huggingface/transformers` (optional dep) was skipped at install or the `@lancedb/lancedb` install was broken — but only AFTER running the full init: `MAX_INIT_RETRIES` (5) attempts of exponential backoff (~31s) per fresh process, spent discovering what `require.resolve` already knows, and every settings surface kept advertising a `KNOWLEDGE_STORE_MODE` that could never run. Now a new execution-free probe (`src/knowledge/availability.ts` — `require.resolve` only, never loads the native modules) is consulted BEFORE the retry loop: `createKnowledgeStoreComponents` returns null immediately (the same clean OFF verdict mode=`none` produces), and `KnowledgeStoreService` records it as DISABLED(`native`) — permanent for the process, never revived by a mode flip, mirroring the existing catch-path semantics. The settings surfaces now tell the truth about it everywhere: `describeKnowledgeStoreMode` gained `available`/`unavailableReason` (flows into `knowledge-config --json` automatically), the `knowledge-config` text output and the `/research-config` TUI's Knowledge Mode item say what is missing and how to repair it, and the healthcheck's 'disabled' verdict names the actual missing packages instead of the generic platform line. One more surface joined them: the `research_knowledge_search` miss message no longer tells a package-gap host the store is "disabled in settings" — a 'native' service disable now reports `store_packages_missing` (the repair is installing the package, not flipping a Knowledge Mode toggle that cannot help), while a true mode disable keeps `store_disabled` and an in-flight/failed init keeps `store_not_ready`. Unit tests pin the probe, the fail-fast (embedder factory never invoked), the native-vs-mode disable distinction, and the miss-reason split.
+
 ## [1.6.8] - 2026-08-31
 
 ### Changed

package/docs/CONFIGURATION.md

@@ -71,6 +71,7 @@ Research
 | `PI_RESEARCH_MAX_SCRAPE_BATCHES` (TUI) | `2` | 0–99 | Scrape batches per researcher (0 = unlimited). When prompt caching is known to be active for the resolved research model (Anthropic API models, or a provider route explicitly configured for Anthropic-style cache control), the effective limit is this value plus one — the cached prompt prefix makes the extra batch cheap. |
 | `PI_RESEARCH_MAX_GATHERING_CALLS` | `12` | 1–100 | Shared web-gathering calls per researcher (`search` + `security_search` + `stackexchange` + `youtube_transcript`). |
 | `PI_RESEARCH_MAX_CONCURRENT_SCRAPES` | `3` | 1–20 | Concurrent URLs fetched per scrape batch. |
+| `PI_RESEARCH_MAX_SCRAPE_URLS` | `8` | 1–20 | Maximum URLs fetched per scrape batch. URLs past the cap are listed under "Not Fetched — Over Batch Cap" and must be requested in a later batch. (Promoted from a hardcoded constant so it is env/config-file tunable like every other scrape knob.) |
 | `PI_RESEARCH_MAX_RETRIES` | `2` | 0–5 | Retries per researcher request. |
 | `PI_RESEARCH_RETRY_DELAY_MS` | `2000` | 100–10000 | Base delay between retries. |
 | `PI_RESEARCH_MAX_FAILED_RESEARCHERS` | `2` | 1–10 | Unique researcher failures that abort the whole run. Raise to let slower, still-in-flight researchers finish before giving up. |
@@ -121,7 +122,7 @@ See the [knowledge store doc](KNOWLEDGE-STORE.md) for what each value does.
 
 | Variable | Default | Range | Description |
 |----------|---------|-------|-------------|
-| `PI_RESEARCH_KNOWLEDGE_STORE_MODE` (TUI) `[project]` | `global` | none · project · global | Store scope: one shared store across every directory (`global`), scoped to the current directory (`project`), or disabled (`none`). |
+| `PI_RESEARCH_KNOWLEDGE_STORE_MODE` (TUI) `[project]` | `global` | none · project · global | Store scope: one shared store across every directory (`global`), scoped to the current directory (`project`), or disabled (`none`). Independently of this setting, the store is cleanly OFF when its required packages are not installed (optional `@huggingface/transformers` skipped at install, broken `@lancedb/lancedb`): init fails fast instead of retry-storming, and `pi-research knowledge-config`, the `/research-config` menu, and the healthcheck all name the missing package and the repair. |
 | `PI_RESEARCH_EMBEDDING_MODEL` (TUI) | `onnx-community/granite-embedding-small-english-r2-ONNX` | — | Embedding model. Changing it clears the store and starts fresh. |
 | `PI_RESEARCH_EMBEDDING_DEVICE` (TUI) | `auto` | auto · webgpu · cpu | Inference backend. `auto` probes WebGPU viability out-of-process and falls back to CPU; `cpu` forces CPU; `webgpu` forces the GPU path with no probe (advanced — can hard-crash on a software GPU). The TUI exposes only `auto` (as "GPU") and `cpu`. |
 | `PI_RESEARCH_CACHE_TTL_DAYS` (TUI) | `30` | 1–365 | How long cached findings are kept before eviction. |

package/docs/KNOWLEDGE-STORE.md

@@ -144,17 +144,25 @@ platform/architecture pairs:
 | Windows | x64 / arm64 | Supported |
 
 Intel Macs (`darwin-x64`) have no prebuilt binary for either component, so the
-knowledge store cannot run there. The degradation is automatic:
+knowledge store cannot run there. The same clean OFF applies on any platform when a
+required package is not installed at all — optional `@huggingface/transformers` is
+skipped by `npm install --omit=optional` (and by an npm optional-dependency bug), or
+the `@lancedb/lancedb` install is broken. The degradation is automatic and fast:
 
 - Research still works. Search, scraping, YouTube transcripts, the security databases,
   Stack Exchange, planning, and synthesis are unaffected — only the store is missing.
-- The health check reports the store as disabled, not unhealthy: its health component
-  returns disabled (native embedding/vector stack unavailable on this platform) with a
-  healthy status, so the missing component does not drag overall health to "unhealthy"
-  or block a quick (depth-0) run.
-- `research_knowledge_search` does no caching or retrieval; every run goes to the live web.
-
-No configuration is needed on an Intel Mac.
+- The store fails fast. Missing packages are detected by resolution before any
+  initialization is attempted, so there is no retry storm — the store simply starts OFF.
+- The settings surfaces say why. `pi-research knowledge-config`, the `/research-config`
+  menu, and the health check name the missing package and the repair (install the
+  optional dependencies) instead of advertising a mode the store cannot honor. A
+  `research_knowledge_search` miss reports the package gap rather than pointing at a
+  settings toggle that cannot help.
+- The health check reports the store as disabled, not unhealthy, so the missing
+  component does not drag overall health to "unhealthy" or block a quick (depth-0) run.
+
+No configuration is needed: install the package (a full install including optional
+dependencies) and the store comes up; a mode flip alone cannot revive it mid-process.
 
 ### Retention and eviction
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.6.8",
+  "version": "1.6.9",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,

package/src/cli.ts

@@ -916,6 +916,7 @@ export async function cmdKnowledgeConfig(kc: NonNullable<ParsedArgs['knowledgeCo
         `  scope:   ${kc.mode === 'none' ? 'disabled here' : kc.mode === 'project' ? 'this directory only' : 'shared across all directories'}\n` +
         `  saved:   per-directory (${cwd})\n` +
         `  db dir:  ${info.dbDir}\n` +
+        (info.available ? '' : `  state:   ${info.unavailableReason}\n`) +
         (overridden
           ? `\nNOTE: the effective mode here is still '${info.mode}' because a higher-precedence ${info.origin} overrides it. Clear that source for this setting to take effect.\n`
           : ''),
@@ -933,6 +934,7 @@ export async function cmdKnowledgeConfig(kc: NonNullable<ParsedArgs['knowledgeCo
       `knowledge store mode: ${info.mode}   (source: ${info.origin})\n` +
       `  scope:   ${info.mode === 'none' ? 'disabled here' : info.mode === 'project' ? 'this directory only' : 'shared across all directories'}\n` +
       `  db dir:  ${info.dbDir}\n` +
+      (info.available ? '' : `  state:   ${info.unavailableReason}\n`) +
       `\nchange it for THIS directory:  ${BINARY_NAME} knowledge-config set <none|project|global>\n` +
       `change the machine-wide default: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> in ${resolvedConfigPaths().configEnv}\n`,
     );

package/src/config.ts

@@ -21,6 +21,7 @@ import { Type, type Static } from 'typebox';
 import { Value } from 'typebox/value';
 import { normalizeWorkspacePath } from './utils/text-utils.ts';
 import { replaceFileSync } from './utils/atomic-replace.ts';
+import { probeKnowledgeStoreAvailability, describeKnowledgeStoreUnavailability } from './knowledge/availability.ts';
 
 /**
  * Validates configuration schema using TypeBox.
@@ -70,8 +71,10 @@ export const ConfigSchema = Type.Object({
   MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1.0, default: 0.15 }),
   /** Estimated tokens per scrape result for planning (default: 2500) */
   AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 10000, default: 2500 }),
-  /** Maximum number of concurrent scrapes (default: 3) */
+  /** Maximum concurrent scrapes (default: 3) */
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -242,6 +245,7 @@ const USER_MIGRATION_KEYS = [
   'PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING',
   'PI_RESEARCH_AVG_TOKENS_PER_SCRAPE',
   'PI_RESEARCH_MAX_CONCURRENT_SCRAPES',
+  'PI_RESEARCH_MAX_SCRAPE_URLS',
   'PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS',
   'PI_RESEARCH_SEARCH_TIMEOUT_MS',
   'PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS',
@@ -594,6 +598,16 @@ export interface KnowledgeStoreModeInfo {
   origin: string;
   /** Physical LanceDB directory backing the store (shared across project/global modes). */
   dbDir: string;
+  /**
+   * Whether the store can actually RUN here — its required packages
+   * (@lancedb/lancedb, optional @huggingface/transformers) resolve. A configured
+   * mode means nothing when this is false: the store is cleanly OFF regardless.
+   * Settings surfaces must show this alongside `mode` so a host that cannot run
+   * the store never advertises a mode it cannot honor.
+   */
+  available: boolean;
+  /** When {@link available} is false: the full human sentence saying what is missing and how to repair it. */
+  unavailableReason: string;
 }
 
 /**
@@ -632,7 +646,27 @@ export function describeKnowledgeStoreMode(
     else if (fileHasKey(getGlobalEnvFilePath())) origin = 'config.env (machine-wide default)';
   }
 
-  return { mode: cfg.KNOWLEDGE_STORE_MODE, origin, dbDir: getDbDir(cfg, cwd) };
+  return {
+    mode: cfg.KNOWLEDGE_STORE_MODE,
+    origin,
+    dbDir: getDbDir(cfg, cwd),
+    ...availabilityFields(),
+  };
+}
+
+/**
+ * The availability half of {@link KnowledgeStoreModeInfo}, computed by the
+ * knowledge module's execution-free package probe. Kept as a helper (not inlined
+ * in every return) so the shape stays defined in exactly one place. Never
+ * throws: the probe catches resolution errors internally and reports them as
+ * `available: false`.
+ */
+function availabilityFields(): { available: boolean; unavailableReason: string } {
+  const availability = probeKnowledgeStoreAvailability();
+  return {
+    available: availability.available,
+    unavailableReason: describeKnowledgeStoreUnavailability(availability),
+  };
 }
 
 /**
@@ -895,6 +929,7 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
     PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: String(config.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING),
     PI_RESEARCH_AVG_TOKENS_PER_SCRAPE: String(config.AVG_TOKENS_PER_SCRAPE),
     PI_RESEARCH_MAX_CONCURRENT_SCRAPES: String(config.MAX_CONCURRENT_SCRAPES),
+    PI_RESEARCH_MAX_SCRAPE_URLS: String(config.MAX_SCRAPE_URLS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS: String(config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS: String(config.YOUTUBE_TRANSCRIPT_TIMEOUT_MS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG: config.YOUTUBE_TRANSCRIPT_LANG,
@@ -1122,6 +1157,7 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
     MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: parseEnvNumber(e, 'PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING', DEFAULTS.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING, 0.05, 1.0),
     AVG_TOKENS_PER_SCRAPE: parseEnvNumber(e, 'PI_RESEARCH_AVG_TOKENS_PER_SCRAPE', DEFAULTS.AVG_TOKENS_PER_SCRAPE, 500, 10000),
     MAX_CONCURRENT_SCRAPES: parseEnvNumber(e, 'PI_RESEARCH_MAX_CONCURRENT_SCRAPES', DEFAULTS.MAX_CONCURRENT_SCRAPES, 1, 20, true),
+    MAX_SCRAPE_URLS: parseEnvNumber(e, 'PI_RESEARCH_MAX_SCRAPE_URLS', DEFAULTS.MAX_SCRAPE_URLS, 1, 20, true),
     YOUTUBE_TRANSCRIPT_MAX_VIDEOS: parseEnvNumber(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS', DEFAULTS.YOUTUBE_TRANSCRIPT_MAX_VIDEOS, 1, 5, true),
     YOUTUBE_TRANSCRIPT_TIMEOUT_MS: parseEnvNumber(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS', DEFAULTS.YOUTUBE_TRANSCRIPT_TIMEOUT_MS, 5000, 120000),
     YOUTUBE_TRANSCRIPT_LANG: parseEnvString(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG', DEFAULTS.YOUTUBE_TRANSCRIPT_LANG)!,

package/src/constants.ts

@@ -71,8 +71,30 @@ export function getMaxScrapeBatches(config?: Config, cachingActive = false): num
   }
 }
 
-/** Maximum URLs to scrape per batch */
-export const MAX_SCRAPE_URLS = 6;
+/**
+ * Maximum URLs fetched per scrape batch — SCHEMA-DEFAULT MIRROR only.
+ *
+ * The configured value lives in the Config schema (`MAX_SCRAPE_URLS`, env
+ * `PI_RESEARCH_MAX_SCRAPE_URLS`); this constant exists so non-config callers
+ * (docs text, test fixtures) and the accessor's fallback below have a name for
+ * the default. Consumers that enforce the cap (src/tools/scrape.ts) MUST go
+ * through {@link getMaxScrapeUrls} with their resolved config so the enforced
+ * cap, the tool schema description, and the over-batch-cap report can never
+ * drift apart.
+ */
+export const MAX_SCRAPE_URLS = 8;
+
+/**
+ * Get the maximum URLs per scrape batch from config (env/config-file driven,
+ * mirrors getMaxScrapeBatches / getMaxGatheringCalls).
+ */
+export function getMaxScrapeUrls(config?: Config): number {
+  try {
+    return (config || getConfig()).MAX_SCRAPE_URLS;
+  } catch {
+    return MAX_SCRAPE_URLS; // Fallback to default
+  }
+}
 
 /**
  * Get the units per researcher for the progress bar.

package/src/healthcheck/index.ts

@@ -15,6 +15,7 @@ import { ServiceNames } from '../core/service-interfaces.ts';
 import type { IStateManager, IHealthRegistryService, IKnowledgeStoreService, ISchedulerService } from '../core/service-interfaces.ts';
 import { healthRegistry as globalHealthRegistry } from './registry.ts';
 import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
+import { probeKnowledgeStoreAvailability } from '../knowledge/availability.ts';
 
 /**
  * Standalone BrowserCapability check, exported so tests can call it directly
@@ -44,6 +45,25 @@ export async function checkBrowserCapability(): Promise<{ healthy: boolean; erro
   }
 }
 
+/**
+ * The diagnostic status for a DISABLED knowledge store whose disable reason is
+ * 'native'. When the availability probe can attribute the gap to a missing
+ * PACKAGE (optional @huggingface/transformers skipped at install, broken
+ * @lancedb/lancedb install), name it — the generic platform line sent users
+ * platform-hunting when the actual fix was one npm install. When every package
+ * resolves, the gap really is the platform's (no prebuilt native binding, e.g.
+ * darwin-x64) and the platform line is the truth. Shared by the idle fast-path
+ * and the full-inspection path so one host always gets ONE verdict regardless
+ * of whether the check is forced.
+ */
+function knowledgeStoreDisabledStatus(): string {
+  const availability = probeKnowledgeStoreAvailability();
+  const detail = availability.available
+    ? 'native embedding/vector stack unavailable on this platform'
+    : `required packages not resolvable — ${availability.missing.join(', ')}`;
+  return `disabled (${detail})`;
+}
+
 /**
  * Register all health checks with a registry
  */
@@ -116,9 +136,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
       if (existing?.lifecycle === ServiceLifecycle.DISABLED) {
         const reason = existing.getDisabledReason();
         return { healthy: true, diagnostic: {
-          status: reason === 'native'
-            ? 'disabled (native embedding/vector stack unavailable on this platform)'
-            : 'disabled',
+          status: reason === 'native' ? knowledgeStoreDisabledStatus() : 'disabled',
         } };
       }
       // A DISPOSED/DISPOSING service is not "ready" — report it accurately without
@@ -150,13 +168,16 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
       return { healthy: true, diagnostic: { status: 'disabled in config' } };
     }
     // The native ML/vector stack ships no prebuilt for some platforms (Intel macOS
-    // / darwin-x64 has no onnxruntime-node binary nor @lancedb native binding). That
-    // is a permanent capability gap, not a fault — report the store as DISABLED
-    // (healthy) so research still runs without the optional cache. Otherwise this
-    // critical component drags overall health to 'unhealthy' and quick (depth-0)
-    // research aborts with "Research cannot start" for any fresh user on such a host.
+    // / darwin-x64 has no onnxruntime-node binary nor @lancedb native binding), and
+    // the optional @huggingface/transformers dep can be skipped at install time
+    // outright. Both are permanent capability gaps, not faults — report the store
+    // as DISABLED (healthy) so research still runs without the optional cache.
+    // Otherwise this critical component drags overall health to 'unhealthy' and
+    // quick (depth-0) research aborts with "Research cannot start" for any fresh
+    // user on such a host. When the gap is a missing PACKAGE (probe-resolvable),
+    // name it — see knowledgeStoreDisabledStatus().
     if (isNativeStackUnavailableError(initError)) {
-      return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
+      return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
     }
     // getService can SUCCEED on a native-unavailable platform: the service memoizes
     // the failure as DISABLED instead of throwing, so initError stays null and the
@@ -166,9 +187,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
     if (service?.lifecycle === ServiceLifecycle.DISABLED) {
       const reason = service.getDisabledReason();
       return { healthy: true, diagnostic: {
-        status: reason === 'native'
-          ? 'disabled (native embedding/vector stack unavailable on this platform)'
-          : 'disabled',
+        status: reason === 'native' ? knowledgeStoreDisabledStatus() : 'disabled',
       } };
     }
     try {
@@ -232,7 +251,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
       // Same platform-capability gap as above, but surfacing only when the store
       // is touched lazily (getStore/getEmbedder load the native binding here).
       if (isNativeStackUnavailableError(e)) {
-        return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
+        return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
       }
       return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
     }

package/src/infrastructure/browser/thread-worker.mjs

@@ -2665,8 +2665,10 @@ var ConfigSchema = Type.Object({
   MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1, default: 0.15 }),
   /** Estimated tokens per scrape result for planning (default: 2500) */
   AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 1e4, default: 2500 }),
-  /** Maximum number of concurrent scrapes (default: 3) */
+  /** Maximum concurrent scrapes (default: 3) */
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */

package/src/infrastructure/knowledge-store-service.ts

@@ -25,6 +25,7 @@ import {
 import { getEmbedder, clearEmbeddingInstance } from './embedding/embedding-factory.ts';
 import { getConfig } from '../config.ts';
 import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
+import { probeKnowledgeStoreAvailability } from '../knowledge/availability.ts';
 
 /**
  * Knowledge Store Service Implementation
@@ -250,9 +251,19 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         }
 
         if (!components) {
-          logger.debug('[KnowledgeStoreService] Knowledge store is disabled. Setting lifecycle to DISABLED.');
+          // `null` now has TWO causes with different revival semantics:
+          //  - KNOWLEDGE_STORE_MODE='none' → 'mode': a user choice, revivable by a
+          //    later /research-config change without a restart.
+          //  - availability probe says the store's packages are not resolvable →
+          //    'native': a host capability gap on this install, permanent for this
+          //    process — memoized like the catch-path below so later getStore()
+          //    calls don't re-run init, and so the healthcheck/config surfaces
+          //    report the package-level reason rather than a user choice the user
+          //    never made.
+          const depsMissing = !probeKnowledgeStoreAvailability().available;
+          logger.debug(`[KnowledgeStoreService] Knowledge store is disabled (${depsMissing ? 'dependencies missing' : 'mode = none'}). Setting lifecycle to DISABLED.`);
           this.lifecycle = ServiceLifecycle.DISABLED;
-          this._disabledReason = 'mode';
+          this._disabledReason = depsMissing ? 'native' : 'mode';
           this._embedder = null;
           this._store = null;
           this._writerQueue = null;

package/src/knowledge/availability.ts

@@ -0,0 +1,118 @@
+/**
+ * Knowledge Store availability probe
+ *
+ * Cheap, execution-free answer to "can the knowledge store possibly run on this
+ * host?" — BEFORE anything tries to build it.
+ *
+ * Why this exists: the store's ML/vector stack lives in two packages that are
+ * not guaranteed to be present at runtime —
+ *   - `@huggingface/transformers` is an OPTIONAL dependency (skipped wholesale by
+ *     `npm install --omit=optional` and by the npm optional-dependency install
+ *     bug, npm/cli#4828 — see transformers-loader.ts and issue #10), and
+ *   - `@lancedb/lancedb` is a regular dependency whose native binding is absent
+ *     on platforms upstream ships no prebuilt for (e.g. darwin-x64).
+ *
+ * Until now a missing package was only discovered by RUNNING the full
+ * embedder+store initialization — which burned MAX_INIT_RETRIES (5) attempts of
+ * exponential backoff (~31s) per init before KnowledgeStoreService memoized the
+ * failure as DISABLED('native'). Correct outcome, terrible path: every fresh
+ * process paid the storm once, and the settings surfaces (research-config TUI,
+ * `knowledge-config` CLI, healthcheck) kept advertising a KNOWLEDGE_STORE_MODE
+ * that could never actually run.
+ *
+ * The probe resolves both modules through `require.resolve` — module RESOLUTION
+ * only; the module body (and its native bindings) is never executed — so it is
+ * safe to call from settings/CLI paths that must not pay the transformers /
+ * lancedb load cost. Resolution catches the dominant real-world failure (the
+ * package directory absent because an optional dep was skipped or the install
+ * is broken); a package that resolves but fails to LOAD its native binding is
+ * the residual case and remains covered by the runtime
+ * isNativeStackUnavailableError → DISABLED('native') path in
+ * knowledge-store-service.ts.
+ *
+ * Deliberately imports NOTHING from the rest of the project beyond
+ * node: builtins — config.ts re-exports its verdict through
+ * describeKnowledgeStoreMode, and config.ts must not gain a dependency on the
+ * knowledge module graph (knowledge/index.ts imports config.ts).
+ */
+import { createRequire } from 'node:module';
+
+/** Result of the availability probe. */
+export interface KnowledgeStoreAvailability {
+  /** True when every required package resolves. */
+  available: boolean;
+  /** Package specifiers whose absence makes the store unusable, in check order. */
+  missing: string[];
+}
+
+/**
+ * The packages the store cannot run without. `@lancedb/lancedb` is the vector
+ * store itself; `@huggingface/transformers` is the only embedding backend
+ * (see transformers-loader.ts — there is no remote-embedding fallback).
+ */
+const REQUIRED_PACKAGES: readonly string[] = ['@lancedb/lancedb', '@huggingface/transformers'];
+
+// Memoized per-process: resolution touches the filesystem on every miss, the
+// answer cannot change within a live process without a reinstall mid-flight,
+// and the probe sits on settings/health paths that run often. Tests reset it
+// via clearAvailabilityCache.
+let _cached: KnowledgeStoreAvailability | null = null;
+
+/** Resolver signature — injectable so tests never need the real filesystem. */
+export type PackageResolver = (specifier: string) => string;
+
+/**
+ * Default resolver: root at THIS module's install tree, so resolution follows
+ * pi-research's own node_modules (where the optional deps would be) rather
+ * than whatever directory the caller happens to run from.
+ */
+const defaultResolver: PackageResolver = (specifier) =>
+  // import.meta.url survives esbuild's ESM output; createRequire only builds a
+  // resolver — it never evaluates the specifier.
+  createRequire(import.meta.url).resolve(specifier);
+
+/**
+ * Probe whether the knowledge store's required packages are resolvable.
+ * Memoized after the first call; {@link clearAvailabilityCache} resets.
+ */
+export function probeKnowledgeStoreAvailability(resolver: PackageResolver = defaultResolver): KnowledgeStoreAvailability {
+  if (cachedIsValidFor(resolver)) return _cached!;
+  const missing: string[] = [];
+  for (const pkg of REQUIRED_PACKAGES) {
+    try {
+      resolver(pkg);
+    } catch {
+      // require.resolve throws ERR_MODULE_NOT_FOUND / ERR_PACKAGE_PATH_NOT_EXPORTED
+      // / MODULE_NOT_FOUND for every "not installable here" shape we care about.
+      missing.push(pkg);
+    }
+  }
+  const result: KnowledgeStoreAvailability = { available: missing.length === 0, missing };
+  if (resolver === defaultResolver) _cached = result;
+  return result;
+}
+
+function cachedIsValidFor(resolver: PackageResolver): boolean {
+  return _cached !== null && resolver === defaultResolver;
+}
+
+/** Reset the memoized probe result (test isolation). */
+export function clearAvailabilityCache(): void {
+  _cached = null;
+}
+
+/**
+ * Human sentence for why the store is unavailable — empty when available.
+ * Used by the settings surfaces (describeKnowledgeStoreMode, the
+ * /research-config TUI, the healthcheck) so every surface words it the same.
+ */
+export function describeKnowledgeStoreUnavailability(availability: KnowledgeStoreAvailability): string {
+  if (availability.available) return '';
+  const parts = availability.missing.map((pkg) =>
+    pkg === '@huggingface/transformers'
+      ? `'${pkg}' (optional embedding dependency — likely skipped at install time)`
+      : `'${pkg}' (broken install — native vector store missing)`,
+  );
+  return `knowledge store is OFF: required package${parts.length > 1 ? 's' : ''} not resolvable — ${parts.join('; ')}. ` +
+    "Repair with a full install of optional dependencies (e.g. 'npm install' including optional deps, or pi install npm:@lincoln504/pi-research without --omit=optional).";
+}

package/src/knowledge/index.ts

@@ -6,6 +6,7 @@ import { Chunker } from './chunker.ts';
 import { getConfig, validateConfig, getDbDir, type Config } from '../config.ts';
 import { logger } from '../logger.ts';
 import { getModelChunkConfig } from './model-config.ts';
+import { probeKnowledgeStoreAvailability, describeKnowledgeStoreUnavailability, type KnowledgeStoreAvailability } from './availability.ts';
 import type { MigrationStrategy } from './migration.ts';
 import type { 
   IEmbedder, 
@@ -49,12 +50,27 @@ export async function createKnowledgeStoreComponents(
   reconnectFactory?: () => Promise<IEmbedder>,
   withLock?: <T>(fn: () => Promise<T>) => Promise<T>,
   config: Config = getConfig(),
-  workspace: string = process.cwd()
+  workspace: string = process.cwd(),
+  availability: KnowledgeStoreAvailability = probeKnowledgeStoreAvailability()
 ): Promise<KnowledgeStoreComponents | null> {
   if (config.KNOWLEDGE_STORE_MODE === 'none') {
     logger.debug('[knowledge] Knowledge store is disabled in configuration');
     return null;
   }
+
+  // Clean settings-level OFF when the store's packages are not even resolvable
+  // (optional transformers skipped at install, broken lancedb install). Before
+  // this pre-flight, a permanently missing package was discovered by RUNNING
+  // init — MAX_INIT_RETRIES (5) attempts of exponential backoff, ~31s, every
+  // fresh process — before the service degraded to DISABLED. The verdict here
+  // is the same OFF the mode='none' switch produces, just caused by the host
+  // instead of the user: return null (never throw), so every existing
+  // null-tolerant consumer treats it exactly like a disabled store.
+  if (!availability.available) {
+    logger.warn(`[knowledge] ${describeKnowledgeStoreUnavailability(availability)}`);
+    return null;
+  }
+
   validateConfig(config);
 
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {

package/src/orchestration/quick-research-orchestrator.ts

@@ -162,7 +162,7 @@ export class QuickResearchOrchestrator {
             `For roughly one in ${this.config.YOUTUBE_QUERY_EVERY_N} of your queries, append the word 'youtube' (e.g. "<topic> explained youtube") — DuckDuckGo rarely surfaces YouTube otherwise, and YouTube links let you read video transcripts.\n` +
             'Your goal is to gather a focused, high-quality pool of initial links.\n\n' +
             '## Scrape\n' +
-            `After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to 4 URLs each).\n` +
+            `After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to ${this.config.MAX_SCRAPE_URLS} URLs each).\n` +
             'Prioritize primary sources and authoritative data.';
         
         // Consume queued steering messages before starting
@@ -187,6 +187,10 @@ export class QuickResearchOrchestrator {
         // turn within this session) reuse it via prompt-cache reads.
         const prompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
             .replace('{{extra_tool_guidelines}}', '- `search`: Perform broad web searches (Round 1 only).')
+            // Config-driven, mirroring researcher-executor: the prompt's per-batch
+            // target can never drift from the cap the scrape tool enforces.
+            // replaceAll — the template states the budget more than once.
+            .replaceAll('{{max_scrape_urls}}', String(this.config.MAX_SCRAPE_URLS))
             // Quick research has no research lead and no next round — see
             // RESEARCHER_DIGEST_SECTION. Its report IS the deliverable.
             .replace('{{digest_section}}', '')

package/src/orchestration/researcher-executor.ts

@@ -203,6 +203,12 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
   // of the retry loop, because nothing in it varies per attempt.
   const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
     .replace('{{extra_tool_guidelines}}', '')
+    // Config-driven, so the prompt's per-batch target can never drift from the
+    // cap the scrape tool actually enforces (getMaxScrapeUrls in scrape.ts).
+    // replaceAll: the template states the budget more than once (Step 2 target
+    // and Step 3's additional-URLs budget) and String.replace would only swap
+    // the first.
+    .replaceAll('{{max_scrape_urls}}', String(config.MAX_SCRAPE_URLS))
     // Function replacer so `$`-bearing content is inserted literally, not as a
     // String.replace substitution pattern.
     .replace('{{digest_section}}', () => RESEARCHER_DIGEST_SECTION)

package/src/prompts/researcher.md

@@ -29,10 +29,10 @@ Do not treat any source as inherently objective. Justify its inclusion through t
 Combine historical URLs and search results into a unified pool. Use previous session summaries as a guide for what to expect. Any knowledge-store results in your user message were retrieved automatically for your goal from the **Project Knowledge Store** and/or **User Knowledge Store** — cite them with that store name as their `Source:`.
 
 ### Step 2: Scrape Round 1
-Identify the 4 most promising URLs and scrape them in one batch (the scrape tool accepts up to 6 per call; 4 is your target per round). Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
+Identify the most promising URLs and scrape them in one batch. The scrape tool accepts up to {{max_scrape_urls}} URLs per call — {{max_scrape_urls}} is your target per round, so plan each batch at the full budget rather than stopping early. Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
 
 ### Step 3: Scrape Round 2 (if needed)
-If your first batch did not yield enough material, scrape up to 4 additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
+If your first batch did not yield enough material, scrape up to {{max_scrape_urls}} additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
 - **Complement** your siblings: if they're covering area X, pick your own URLs that go deeper on X or fill adjacent gaps.
 - **Diversify**: if you have sources on an angle no one else is touching, prioritize those to maximize session coverage.
 - **Stay on mission**: regardless of what siblings are doing, make sure you have enough material for your assigned goal.

package/src/research-config.ts

@@ -35,6 +35,7 @@ import type { Theme } from './types/research-panel-types.ts';
 // Import the constant directly from its source module rather than the knowledge
 // barrel (./knowledge/index.ts), which would transitively load lancedb.
 import { SUPPORTED_MODELS } from './knowledge/model-config.ts';
+import { probeKnowledgeStoreAvailability, describeKnowledgeStoreUnavailability } from './knowledge/availability.ts';
 import { metrics } from './utils/metrics.ts';
 import {
   extractRunStats,
@@ -142,7 +143,19 @@ async function showInteractiveMenu(ctx: ExtensionContext, pi: ExtensionAPI): Pro
     {
       id: 'KNOWLEDGE_STORE_MODE',
       label: 'Knowledge Mode [project]',
-      description: 'Knowledge store scope — none (disabled), project (scope knowledge per this directory), or global (scope knowledge globally, shared across all projects).\n[project] means configured independently per directory.',
+      description: (() => {
+        const base = 'Knowledge store scope — none (disabled), project (scope knowledge per this directory), or global (scope knowledge globally, shared across all projects).\n[project] means configured independently per directory.';
+        // The configured mode is a lie when the host cannot run the store at all
+        // (optional embedding dep skipped at install, broken lancedb). Surface the
+        // package-level OFF right in the setting, so the menu never advertises a
+        // scope the store cannot honor and the repair path is named where the
+        // user is looking.
+        const availability = probeKnowledgeStoreAvailability();
+        if (!availability.available) {
+          return `${base}\n\n⚠ ${describeKnowledgeStoreUnavailability(availability)}\nThe store stays off — regardless of this setting — until that package is installed.`;
+        }
+        return base;
+      })(),
       currentValue: config.KNOWLEDGE_STORE_MODE,
       values: ['none', 'project', 'global'],
     },

package/src/tools/research-knowledge-search.ts

@@ -840,6 +840,9 @@ function missResult(reason: string): AgentToolResult<unknown> {
     case 'store_disabled':
       text = 'No results found (knowledge store is disabled in settings). Live research can get the info.';
       break;
+    case 'store_packages_missing':
+      text = 'No results found (knowledge store is off — its required packages are not installed on this host, so it stays off regardless of settings). Live research can get the info.';
+      break;
     case 'store_not_ready':
       text = 'No results found (knowledge store is initializing). Live research can get the info.';
       break;
@@ -951,10 +954,16 @@ export function createResearchKnowledgeSearchTool(iface?: ConfigInterface): Tool
         // or null when the store is genuinely disabled/failed.
         const store = await storeService.getStore();
         if (!store) {
-          // null = disabled (mode still 'none') or a genuine init failure — distinguish for the
-          // user-facing message so a real failure doesn't read as "disabled in settings".
-          const lifecycle = (storeService as any).lifecycle;
-          return missResult(lifecycle === 'disabled' ? 'store_disabled' : 'store_not_ready');
+          // null = disabled (a settings choice OR a host whose store packages are
+          // missing) or a genuine init failure — distinguish all three in the
+          // user-facing message. A 'native' disable must not read as "disabled in
+          // settings": the repair is installing the package, not a Knowledge Mode
+          // toggle that cannot help (same truth-telling as knowledge-config, the
+          // /research-config menu, and the healthcheck).
+          const disabledReason = storeService.getDisabledReason();
+          if (disabledReason === 'native') return missResult('store_packages_missing');
+          if (disabledReason === 'mode') return missResult('store_disabled');
+          return missResult('store_not_ready');
         }
 
         const count = await store.count();

package/src/tools/scrape.ts

@@ -17,9 +17,9 @@ import type { ToolUsageTracker } from '../utils/tool-usage-tracker.ts';
 import type { SystemResearchState } from '../orchestration/deep-research-types.ts';
 import { deduplicateUrls, normalizeUrl, getCachedScrapedContent, registerResearcherScrapes, buildSessionPoolFooter } from '../utils/shared-links.ts';
 import {
-  MAX_SCRAPE_URLS,
   BATCH_2_DEFAULT_CONCURRENCY,
   getMaxScrapeBatches,
+  getMaxScrapeUrls,
   DEFAULT_MODEL_CONTEXT_WINDOW,
   MAX_SCRAPE_CONTENT_CHARS_PER_DOC,
 } from '../constants.ts';
@@ -57,6 +57,10 @@ export function createScrapeTool(options: {
   // enforced — a real, if harmless, missed-opportunity: the model would stop one
   // batch short of what it was actually allowed to use.
   const maxScrapeBatches = options.tracker?.getToolLimit('scrape') ?? getMaxScrapeBatches(config);
+  // One resolved value feeds the tool schema text, the batch-protocol text, the
+  // per-call slice, and the over-cap report — the enforced cap and every string
+  // the model sees are the same number by construction.
+  const maxScrapeUrls = getMaxScrapeUrls(config);
   const container = tryGetServiceContainerFromCtx(options.ctx);
 
   // Fallback global state when no orchestration context is provided
@@ -78,7 +82,7 @@ export function createScrapeTool(options: {
   });
 
   const ScrapeParamsSchema = Type.Object({
-    urls: Type.Array(Type.String({ description: `The URLs to scrape (only the first ${MAX_SCRAPE_URLS} are fetched per call; extras are reported under "Not Fetched" rather than silently dropped)` }), { minItems: 1, maxItems: 20 }),
+    urls: Type.Array(Type.String({ description: `The URLs to scrape (only the first ${maxScrapeUrls} are fetched per call; extras are reported under "Not Fetched" rather than silently dropped)` }), { minItems: 1, maxItems: 20 }),
     maxConcurrency: Type.Optional(Type.Number({ default: config.MAX_CONCURRENT_SCRAPES, minimum: 1, maximum: 20 })),
   });
 
@@ -92,7 +96,7 @@ export function createScrapeTool(options: {
     batchProtocolText = `PROTOCOL: Batch 1 → Batch 2 → ... (up to ${effectiveLimit} batches)`;
   } else {
     const batchNumbers = Array.from({ length: effectiveLimit }, (_, i) => `Batch ${i + 1}`).join(' → ');
-    batchProtocolText = `PROTOCOL: ${batchNumbers} (up to ${MAX_SCRAPE_URLS} URLs each).`;
+    batchProtocolText = `PROTOCOL: ${batchNumbers} (up to ${maxScrapeUrls} URLs each).`;
   }
 
   return {
@@ -102,7 +106,7 @@ export function createScrapeTool(options: {
     promptSnippet: `Read full content of web pages or PDFs (up to ${maxScrapeBatches} batches)`,
     promptGuidelines: [
       batchProtocolText,
-      `Up to ${MAX_SCRAPE_URLS} URLs per batch.`,
+      `Up to ${maxScrapeUrls} URLs per batch.`,
       'Handshake is ELIMINATED. Start scraping immediately.',
       'PDFs are auto-detected and extracted with high fidelity.',
     ],
@@ -175,8 +179,8 @@ export function createScrapeTool(options: {
       // in results nor in Failed-to-Scrape, and the session-pool footer excludes the
       // current batch).
       const fetchCandidates = [...dedupedUrls, ...evictedDuplicates];
-      const finalUrls = fetchCandidates.slice(0, MAX_SCRAPE_URLS);
-      const skippedUrls = fetchCandidates.slice(MAX_SCRAPE_URLS);
+      const finalUrls = fetchCandidates.slice(0, maxScrapeUrls);
+      const skippedUrls = fetchCandidates.slice(maxScrapeUrls);
 
       // Context gate: block if projected token usage would exceed threshold
       if (options.getTokensUsed) {
@@ -364,7 +368,7 @@ export function createScrapeTool(options: {
 
       if (skippedUrls.length > 0) {
         markdown += `## Not Fetched — Over Batch Cap (${skippedUrls.length})\n\n`;
-        markdown += `Only the first ${MAX_SCRAPE_URLS} URLs of a batch are fetched. The following URLs were NOT fetched — no content for them appears above. Request them in a later scrape batch if still needed:\n`;
+        markdown += `Only the first ${maxScrapeUrls} URLs of a batch are fetched. The following URLs were NOT fetched — no content for them appears above. Request them in a later scrape batch if still needed:\n`;
         for (const url of skippedUrls) {
           markdown += `- ${url}\n`;
         }