@lincoln504/pi-research 1.6.8 → 1.6.10

package/src/config.ts

@@ -21,6 +21,7 @@ import { Type, type Static } from 'typebox';
 import { Value } from 'typebox/value';
 import { normalizeWorkspacePath } from './utils/text-utils.ts';
 import { replaceFileSync } from './utils/atomic-replace.ts';
+import { probeKnowledgeStoreAvailability, describeKnowledgeStoreUnavailability } from './knowledge/availability.ts';
 
 /**
  * Validates configuration schema using TypeBox.
@@ -70,8 +71,10 @@ export const ConfigSchema = Type.Object({
   MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1.0, default: 0.15 }),
   /** Estimated tokens per scrape result for planning (default: 2500) */
   AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 10000, default: 2500 }),
-  /** Maximum number of concurrent scrapes (default: 3) */
+  /** Maximum concurrent scrapes (default: 3) */
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -242,6 +245,7 @@ const USER_MIGRATION_KEYS = [
   'PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING',
   'PI_RESEARCH_AVG_TOKENS_PER_SCRAPE',
   'PI_RESEARCH_MAX_CONCURRENT_SCRAPES',
+  'PI_RESEARCH_MAX_SCRAPE_URLS',
   'PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS',
   'PI_RESEARCH_SEARCH_TIMEOUT_MS',
   'PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS',
@@ -594,6 +598,16 @@ export interface KnowledgeStoreModeInfo {
   origin: string;
   /** Physical LanceDB directory backing the store (shared across project/global modes). */
   dbDir: string;
+  /**
+   * Whether the store can actually RUN here — its required packages
+   * (@lancedb/lancedb, optional @huggingface/transformers) resolve. A configured
+   * mode means nothing when this is false: the store is cleanly OFF regardless.
+   * Settings surfaces must show this alongside `mode` so a host that cannot run
+   * the store never advertises a mode it cannot honor.
+   */
+  available: boolean;
+  /** When {@link available} is false: the full human sentence saying what is missing and how to repair it. */
+  unavailableReason: string;
 }
 
 /**
@@ -632,7 +646,27 @@ export function describeKnowledgeStoreMode(
     else if (fileHasKey(getGlobalEnvFilePath())) origin = 'config.env (machine-wide default)';
   }
 
-  return { mode: cfg.KNOWLEDGE_STORE_MODE, origin, dbDir: getDbDir(cfg, cwd) };
+  return {
+    mode: cfg.KNOWLEDGE_STORE_MODE,
+    origin,
+    dbDir: getDbDir(cfg, cwd),
+    ...availabilityFields(),
+  };
+}
+
+/**
+ * The availability half of {@link KnowledgeStoreModeInfo}, computed by the
+ * knowledge module's execution-free package probe. Kept as a helper (not inlined
+ * in every return) so the shape stays defined in exactly one place. Never
+ * throws: the probe catches resolution errors internally and reports them as
+ * `available: false`.
+ */
+function availabilityFields(): { available: boolean; unavailableReason: string } {
+  const availability = probeKnowledgeStoreAvailability();
+  return {
+    available: availability.available,
+    unavailableReason: describeKnowledgeStoreUnavailability(availability),
+  };
 }
 
 /**
@@ -895,6 +929,7 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
     PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: String(config.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING),
     PI_RESEARCH_AVG_TOKENS_PER_SCRAPE: String(config.AVG_TOKENS_PER_SCRAPE),
     PI_RESEARCH_MAX_CONCURRENT_SCRAPES: String(config.MAX_CONCURRENT_SCRAPES),
+    PI_RESEARCH_MAX_SCRAPE_URLS: String(config.MAX_SCRAPE_URLS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS: String(config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS: String(config.YOUTUBE_TRANSCRIPT_TIMEOUT_MS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG: config.YOUTUBE_TRANSCRIPT_LANG,
@@ -1122,6 +1157,7 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
     MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: parseEnvNumber(e, 'PI_RESEARCH_MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING', DEFAULTS.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING, 0.05, 1.0),
     AVG_TOKENS_PER_SCRAPE: parseEnvNumber(e, 'PI_RESEARCH_AVG_TOKENS_PER_SCRAPE', DEFAULTS.AVG_TOKENS_PER_SCRAPE, 500, 10000),
     MAX_CONCURRENT_SCRAPES: parseEnvNumber(e, 'PI_RESEARCH_MAX_CONCURRENT_SCRAPES', DEFAULTS.MAX_CONCURRENT_SCRAPES, 1, 20, true),
+    MAX_SCRAPE_URLS: parseEnvNumber(e, 'PI_RESEARCH_MAX_SCRAPE_URLS', DEFAULTS.MAX_SCRAPE_URLS, 1, 20, true),
     YOUTUBE_TRANSCRIPT_MAX_VIDEOS: parseEnvNumber(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS', DEFAULTS.YOUTUBE_TRANSCRIPT_MAX_VIDEOS, 1, 5, true),
     YOUTUBE_TRANSCRIPT_TIMEOUT_MS: parseEnvNumber(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS', DEFAULTS.YOUTUBE_TRANSCRIPT_TIMEOUT_MS, 5000, 120000),
     YOUTUBE_TRANSCRIPT_LANG: parseEnvString(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG', DEFAULTS.YOUTUBE_TRANSCRIPT_LANG)!,

package/src/constants.ts

@@ -71,8 +71,30 @@ export function getMaxScrapeBatches(config?: Config, cachingActive = false): num
   }
 }
 
-/** Maximum URLs to scrape per batch */
-export const MAX_SCRAPE_URLS = 6;
+/**
+ * Maximum URLs fetched per scrape batch — SCHEMA-DEFAULT MIRROR only.
+ *
+ * The configured value lives in the Config schema (`MAX_SCRAPE_URLS`, env
+ * `PI_RESEARCH_MAX_SCRAPE_URLS`); this constant exists so non-config callers
+ * (docs text, test fixtures) and the accessor's fallback below have a name for
+ * the default. Consumers that enforce the cap (src/tools/scrape.ts) MUST go
+ * through {@link getMaxScrapeUrls} with their resolved config so the enforced
+ * cap, the tool schema description, and the over-batch-cap report can never
+ * drift apart.
+ */
+export const MAX_SCRAPE_URLS = 8;
+
+/**
+ * Get the maximum URLs per scrape batch from config (env/config-file driven,
+ * mirrors getMaxScrapeBatches / getMaxGatheringCalls).
+ */
+export function getMaxScrapeUrls(config?: Config): number {
+  try {
+    return (config || getConfig()).MAX_SCRAPE_URLS;
+  } catch {
+    return MAX_SCRAPE_URLS; // Fallback to default
+  }
+}
 
 /**
  * Get the units per researcher for the progress bar.

package/src/healthcheck/index.ts

@@ -15,6 +15,7 @@ import { ServiceNames } from '../core/service-interfaces.ts';
 import type { IStateManager, IHealthRegistryService, IKnowledgeStoreService, ISchedulerService } from '../core/service-interfaces.ts';
 import { healthRegistry as globalHealthRegistry } from './registry.ts';
 import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
+import { probeKnowledgeStoreAvailability } from '../knowledge/availability.ts';
 
 /**
  * Standalone BrowserCapability check, exported so tests can call it directly
@@ -44,6 +45,25 @@ export async function checkBrowserCapability(): Promise<{ healthy: boolean; erro
   }
 }
 
+/**
+ * The diagnostic status for a DISABLED knowledge store whose disable reason is
+ * 'native'. When the availability probe can attribute the gap to a missing
+ * PACKAGE (optional @huggingface/transformers skipped at install, broken
+ * @lancedb/lancedb install), name it — the generic platform line sent users
+ * platform-hunting when the actual fix was one npm install. When every package
+ * resolves, the gap really is the platform's (no prebuilt native binding, e.g.
+ * darwin-x64) and the platform line is the truth. Shared by the idle fast-path
+ * and the full-inspection path so one host always gets ONE verdict regardless
+ * of whether the check is forced.
+ */
+function knowledgeStoreDisabledStatus(): string {
+  const availability = probeKnowledgeStoreAvailability();
+  const detail = availability.available
+    ? 'native embedding/vector stack unavailable on this platform'
+    : `required packages not resolvable — ${availability.missing.join(', ')}`;
+  return `disabled (${detail})`;
+}
+
 /**
  * Register all health checks with a registry
  */
@@ -116,9 +136,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
       if (existing?.lifecycle === ServiceLifecycle.DISABLED) {
         const reason = existing.getDisabledReason();
         return { healthy: true, diagnostic: {
-          status: reason === 'native'
-            ? 'disabled (native embedding/vector stack unavailable on this platform)'
-            : 'disabled',
+          status: reason === 'native' ? knowledgeStoreDisabledStatus() : 'disabled',
         } };
       }
       // A DISPOSED/DISPOSING service is not "ready" — report it accurately without
@@ -150,13 +168,16 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
       return { healthy: true, diagnostic: { status: 'disabled in config' } };
     }
     // The native ML/vector stack ships no prebuilt for some platforms (Intel macOS
-    // / darwin-x64 has no onnxruntime-node binary nor @lancedb native binding). That
-    // is a permanent capability gap, not a fault — report the store as DISABLED
-    // (healthy) so research still runs without the optional cache. Otherwise this
-    // critical component drags overall health to 'unhealthy' and quick (depth-0)
-    // research aborts with "Research cannot start" for any fresh user on such a host.
+    // / darwin-x64 has no onnxruntime-node binary nor @lancedb native binding), and
+    // the optional @huggingface/transformers dep can be skipped at install time
+    // outright. Both are permanent capability gaps, not faults — report the store
+    // as DISABLED (healthy) so research still runs without the optional cache.
+    // Otherwise this critical component drags overall health to 'unhealthy' and
+    // quick (depth-0) research aborts with "Research cannot start" for any fresh
+    // user on such a host. When the gap is a missing PACKAGE (probe-resolvable),
+    // name it — see knowledgeStoreDisabledStatus().
     if (isNativeStackUnavailableError(initError)) {
-      return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
+      return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
     }
     // getService can SUCCEED on a native-unavailable platform: the service memoizes
     // the failure as DISABLED instead of throwing, so initError stays null and the
@@ -166,9 +187,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
     if (service?.lifecycle === ServiceLifecycle.DISABLED) {
       const reason = service.getDisabledReason();
       return { healthy: true, diagnostic: {
-        status: reason === 'native'
-          ? 'disabled (native embedding/vector stack unavailable on this platform)'
-          : 'disabled',
+        status: reason === 'native' ? knowledgeStoreDisabledStatus() : 'disabled',
       } };
     }
     try {
@@ -232,7 +251,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
       // Same platform-capability gap as above, but surfacing only when the store
       // is touched lazily (getStore/getEmbedder load the native binding here).
       if (isNativeStackUnavailableError(e)) {
-        return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
+        return { healthy: true, diagnostic: { status: knowledgeStoreDisabledStatus() } };
       }
       return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
     }

package/src/infrastructure/browser/thread-worker.mjs

@@ -2665,8 +2665,10 @@ var ConfigSchema = Type.Object({
   MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING: Type.Number({ minimum: 0.05, maximum: 1, default: 0.15 }),
   /** Estimated tokens per scrape result for planning (default: 2500) */
   AVG_TOKENS_PER_SCRAPE: Type.Number({ minimum: 500, maximum: 1e4, default: 2500 }),
-  /** Maximum number of concurrent scrapes (default: 3) */
+  /** Maximum concurrent scrapes (default: 3) */
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
+  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
+  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */

package/src/infrastructure/knowledge-store-service.ts

@@ -25,6 +25,7 @@ import {
 import { getEmbedder, clearEmbeddingInstance } from './embedding/embedding-factory.ts';
 import { getConfig } from '../config.ts';
 import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
+import { probeKnowledgeStoreAvailability } from '../knowledge/availability.ts';
 
 /**
  * Knowledge Store Service Implementation
@@ -250,9 +251,19 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         }
 
         if (!components) {
-          logger.debug('[KnowledgeStoreService] Knowledge store is disabled. Setting lifecycle to DISABLED.');
+          // `null` now has TWO causes with different revival semantics:
+          //  - KNOWLEDGE_STORE_MODE='none' → 'mode': a user choice, revivable by a
+          //    later /research-config change without a restart.
+          //  - availability probe says the store's packages are not resolvable →
+          //    'native': a host capability gap on this install, permanent for this
+          //    process — memoized like the catch-path below so later getStore()
+          //    calls don't re-run init, and so the healthcheck/config surfaces
+          //    report the package-level reason rather than a user choice the user
+          //    never made.
+          const depsMissing = !probeKnowledgeStoreAvailability().available;
+          logger.debug(`[KnowledgeStoreService] Knowledge store is disabled (${depsMissing ? 'dependencies missing' : 'mode = none'}). Setting lifecycle to DISABLED.`);
           this.lifecycle = ServiceLifecycle.DISABLED;
-          this._disabledReason = 'mode';
+          this._disabledReason = depsMissing ? 'native' : 'mode';
           this._embedder = null;
           this._store = null;
           this._writerQueue = null;

package/src/knowledge/availability.ts

@@ -0,0 +1,118 @@
+/**
+ * Knowledge Store availability probe
+ *
+ * Cheap, execution-free answer to "can the knowledge store possibly run on this
+ * host?" — BEFORE anything tries to build it.
+ *
+ * Why this exists: the store's ML/vector stack lives in two packages that are
+ * not guaranteed to be present at runtime —
+ *   - `@huggingface/transformers` is an OPTIONAL dependency (skipped wholesale by
+ *     `npm install --omit=optional` and by the npm optional-dependency install
+ *     bug, npm/cli#4828 — see transformers-loader.ts and issue #10), and
+ *   - `@lancedb/lancedb` is a regular dependency whose native binding is absent
+ *     on platforms upstream ships no prebuilt for (e.g. darwin-x64).
+ *
+ * Until now a missing package was only discovered by RUNNING the full
+ * embedder+store initialization — which burned MAX_INIT_RETRIES (5) attempts of
+ * exponential backoff (~31s) per init before KnowledgeStoreService memoized the
+ * failure as DISABLED('native'). Correct outcome, terrible path: every fresh
+ * process paid the storm once, and the settings surfaces (research-config TUI,
+ * `knowledge-config` CLI, healthcheck) kept advertising a KNOWLEDGE_STORE_MODE
+ * that could never actually run.
+ *
+ * The probe resolves both modules through `require.resolve` — module RESOLUTION
+ * only; the module body (and its native bindings) is never executed — so it is
+ * safe to call from settings/CLI paths that must not pay the transformers /
+ * lancedb load cost. Resolution catches the dominant real-world failure (the
+ * package directory absent because an optional dep was skipped or the install
+ * is broken); a package that resolves but fails to LOAD its native binding is
+ * the residual case and remains covered by the runtime
+ * isNativeStackUnavailableError → DISABLED('native') path in
+ * knowledge-store-service.ts.
+ *
+ * Deliberately imports NOTHING from the rest of the project beyond
+ * node: builtins — config.ts re-exports its verdict through
+ * describeKnowledgeStoreMode, and config.ts must not gain a dependency on the
+ * knowledge module graph (knowledge/index.ts imports config.ts).
+ */
+import { createRequire } from 'node:module';
+
+/** Result of the availability probe. */
+export interface KnowledgeStoreAvailability {
+  /** True when every required package resolves. */
+  available: boolean;
+  /** Package specifiers whose absence makes the store unusable, in check order. */
+  missing: string[];
+}
+
+/**
+ * The packages the store cannot run without. `@lancedb/lancedb` is the vector
+ * store itself; `@huggingface/transformers` is the only embedding backend
+ * (see transformers-loader.ts — there is no remote-embedding fallback).
+ */
+const REQUIRED_PACKAGES: readonly string[] = ['@lancedb/lancedb', '@huggingface/transformers'];
+
+// Memoized per-process: resolution touches the filesystem on every miss, the
+// answer cannot change within a live process without a reinstall mid-flight,
+// and the probe sits on settings/health paths that run often. Tests reset it
+// via clearAvailabilityCache.
+let _cached: KnowledgeStoreAvailability | null = null;
+
+/** Resolver signature — injectable so tests never need the real filesystem. */
+export type PackageResolver = (specifier: string) => string;
+
+/**
+ * Default resolver: root at THIS module's install tree, so resolution follows
+ * pi-research's own node_modules (where the optional deps would be) rather
+ * than whatever directory the caller happens to run from.
+ */
+const defaultResolver: PackageResolver = (specifier) =>
+  // import.meta.url survives esbuild's ESM output; createRequire only builds a
+  // resolver — it never evaluates the specifier.
+  createRequire(import.meta.url).resolve(specifier);
+
+/**
+ * Probe whether the knowledge store's required packages are resolvable.
+ * Memoized after the first call; {@link clearAvailabilityCache} resets.
+ */
+export function probeKnowledgeStoreAvailability(resolver: PackageResolver = defaultResolver): KnowledgeStoreAvailability {
+  if (cachedIsValidFor(resolver)) return _cached!;
+  const missing: string[] = [];
+  for (const pkg of REQUIRED_PACKAGES) {
+    try {
+      resolver(pkg);
+    } catch {
+      // require.resolve throws ERR_MODULE_NOT_FOUND / ERR_PACKAGE_PATH_NOT_EXPORTED
+      // / MODULE_NOT_FOUND for every "not installable here" shape we care about.
+      missing.push(pkg);
+    }
+  }
+  const result: KnowledgeStoreAvailability = { available: missing.length === 0, missing };
+  if (resolver === defaultResolver) _cached = result;
+  return result;
+}
+
+function cachedIsValidFor(resolver: PackageResolver): boolean {
+  return _cached !== null && resolver === defaultResolver;
+}
+
+/** Reset the memoized probe result (test isolation). */
+export function clearAvailabilityCache(): void {
+  _cached = null;
+}
+
+/**
+ * Human sentence for why the store is unavailable — empty when available.
+ * Used by the settings surfaces (describeKnowledgeStoreMode, the
+ * /research-config TUI, the healthcheck) so every surface words it the same.
+ */
+export function describeKnowledgeStoreUnavailability(availability: KnowledgeStoreAvailability): string {
+  if (availability.available) return '';
+  const parts = availability.missing.map((pkg) =>
+    pkg === '@huggingface/transformers'
+      ? `'${pkg}' (optional embedding dependency — likely skipped at install time)`
+      : `'${pkg}' (broken install — native vector store missing)`,
+  );
+  return `knowledge store is OFF: required package${parts.length > 1 ? 's' : ''} not resolvable — ${parts.join('; ')}. ` +
+    "Repair with a full install of optional dependencies (e.g. 'npm install' including optional deps, or pi install npm:@lincoln504/pi-research without --omit=optional).";
+}

package/src/knowledge/index.ts

@@ -6,6 +6,7 @@ import { Chunker } from './chunker.ts';
 import { getConfig, validateConfig, getDbDir, type Config } from '../config.ts';
 import { logger } from '../logger.ts';
 import { getModelChunkConfig } from './model-config.ts';
+import { probeKnowledgeStoreAvailability, describeKnowledgeStoreUnavailability, type KnowledgeStoreAvailability } from './availability.ts';
 import type { MigrationStrategy } from './migration.ts';
 import type { 
   IEmbedder, 
@@ -49,12 +50,27 @@ export async function createKnowledgeStoreComponents(
   reconnectFactory?: () => Promise<IEmbedder>,
   withLock?: <T>(fn: () => Promise<T>) => Promise<T>,
   config: Config = getConfig(),
-  workspace: string = process.cwd()
+  workspace: string = process.cwd(),
+  availability: KnowledgeStoreAvailability = probeKnowledgeStoreAvailability()
 ): Promise<KnowledgeStoreComponents | null> {
   if (config.KNOWLEDGE_STORE_MODE === 'none') {
     logger.debug('[knowledge] Knowledge store is disabled in configuration');
     return null;
   }
+
+  // Clean settings-level OFF when the store's packages are not even resolvable
+  // (optional transformers skipped at install, broken lancedb install). Before
+  // this pre-flight, a permanently missing package was discovered by RUNNING
+  // init — MAX_INIT_RETRIES (5) attempts of exponential backoff, ~31s, every
+  // fresh process — before the service degraded to DISABLED. The verdict here
+  // is the same OFF the mode='none' switch produces, just caused by the host
+  // instead of the user: return null (never throw), so every existing
+  // null-tolerant consumer treats it exactly like a disabled store.
+  if (!availability.available) {
+    logger.warn(`[knowledge] ${describeKnowledgeStoreUnavailability(availability)}`);
+    return null;
+  }
+
   validateConfig(config);
 
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {

package/src/orchestration/quick-research-orchestrator.ts

@@ -162,7 +162,7 @@ export class QuickResearchOrchestrator {
             `For roughly one in ${this.config.YOUTUBE_QUERY_EVERY_N} of your queries, append the word 'youtube' (e.g. "<topic> explained youtube") — DuckDuckGo rarely surfaces YouTube otherwise, and YouTube links let you read video transcripts.\n` +
             'Your goal is to gather a focused, high-quality pool of initial links.\n\n' +
             '## Scrape\n' +
-            `After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to 4 URLs each).\n` +
+            `After searching, scrape the best sources using the \`scrape\` tool (up to ${maxScrapeBatchesDisplay} batches, up to ${this.config.MAX_SCRAPE_URLS} URLs each).\n` +
             'Prioritize primary sources and authoritative data.';
         
         // Consume queued steering messages before starting
@@ -187,6 +187,10 @@ export class QuickResearchOrchestrator {
         // turn within this session) reuse it via prompt-cache reads.
         const prompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
             .replace('{{extra_tool_guidelines}}', '- `search`: Perform broad web searches (Round 1 only).')
+            // Config-driven, mirroring researcher-executor: the prompt's per-batch
+            // target can never drift from the cap the scrape tool enforces.
+            // replaceAll — the template states the budget more than once.
+            .replaceAll('{{max_scrape_urls}}', String(this.config.MAX_SCRAPE_URLS))
             // Quick research has no research lead and no next round — see
             // RESEARCHER_DIGEST_SECTION. Its report IS the deliverable.
             .replace('{{digest_section}}', '')

package/src/orchestration/researcher-executor.ts

@@ -203,6 +203,12 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
   // of the retry loop, because nothing in it varies per attempt.
   const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
     .replace('{{extra_tool_guidelines}}', '')
+    // Config-driven, so the prompt's per-batch target can never drift from the
+    // cap the scrape tool actually enforces (getMaxScrapeUrls in scrape.ts).
+    // replaceAll: the template states the budget more than once (Step 2 target
+    // and Step 3's additional-URLs budget) and String.replace would only swap
+    // the first.
+    .replaceAll('{{max_scrape_urls}}', String(config.MAX_SCRAPE_URLS))
     // Function replacer so `$`-bearing content is inserted literally, not as a
     // String.replace substitution pattern.
     .replace('{{digest_section}}', () => RESEARCHER_DIGEST_SECTION)

package/src/prompts/researcher.md

@@ -29,10 +29,10 @@ Do not treat any source as inherently objective. Justify its inclusion through t
 Combine historical URLs and search results into a unified pool. Use previous session summaries as a guide for what to expect. Any knowledge-store results in your user message were retrieved automatically for your goal from the **Project Knowledge Store** and/or **User Knowledge Store** — cite them with that store name as their `Source:`.
 
 ### Step 2: Scrape Round 1
-Identify the 4 most promising URLs and scrape them in one batch (the scrape tool accepts up to 6 per call; 4 is your target per round). Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
+Identify the most promising URLs and scrape them in one batch. The scrape tool accepts up to {{max_scrape_urls}} URLs per call — {{max_scrape_urls}} is your target per round, so plan each batch at the full budget rather than stopping early. Prioritize primary sources, authoritative references, and dense documentation; all else being equal, lean toward more recent / current sources and topics.
 
 ### Step 3: Scrape Round 2 (if needed)
-If your first batch did not yield enough material, scrape up to 4 additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
+If your first batch did not yield enough material, scrape up to {{max_scrape_urls}} additional URLs from **your own source list** that you haven't read yet. Before choosing, review the **Session URL Pool** at the bottom of each scrape response — it shows what topics and domains your sibling researchers are already covering. Use this to steer your remaining scrapes from your own list:
 - **Complement** your siblings: if they're covering area X, pick your own URLs that go deeper on X or fill adjacent gaps.
 - **Diversify**: if you have sources on an angle no one else is touching, prioritize those to maximize session coverage.
 - **Stay on mission**: regardless of what siblings are doing, make sure you have enough material for your assigned goal.

package/src/research-config.ts

@@ -35,6 +35,7 @@ import type { Theme } from './types/research-panel-types.ts';
 // Import the constant directly from its source module rather than the knowledge
 // barrel (./knowledge/index.ts), which would transitively load lancedb.
 import { SUPPORTED_MODELS } from './knowledge/model-config.ts';
+import { probeKnowledgeStoreAvailability, describeKnowledgeStoreUnavailability } from './knowledge/availability.ts';
 import { metrics } from './utils/metrics.ts';
 import {
   extractRunStats,
@@ -88,12 +89,34 @@ const ENV_KEY_BY_SETTING_ID: Record<string, string> = {
  * Main command handler for /research-config
  */
 export async function handleResearchConfigCommand(
-  _args: string,
+  args: string,
   ctx: ExtensionContext,
   pi: ExtensionAPI
 ): Promise<void> {
-  if (!ctx.hasUI) {
-    ctx.ui.notify('Interactive menu requires UI mode', 'error');
+  const verb = args.trim().split(/\s+/)[0]?.toLowerCase() ?? '';
+  if (verb === 'health') {
+    await runHealthCheckAction(ctx, pi);
+    return;
+  }
+  if (verb === 'knowledge-status' || verb === 'knowledge_status') {
+    await showKnowledgeStatusAction(ctx, pi);
+    return;
+  }
+  if (ctx.mode !== 'tui' || !ctx.hasUI) {
+    // The interactive menu needs pi's real TUI. In RPC/web-hub/print/json/SDK hosts
+    // ctx.hasUI is true (RPC) and the menu cannot render — awaiting it silently
+    // nudges the host toward a hang and at best does nothing visible. Give a clear
+    // non-interactive answer with the headless diagnostics that DO work here.
+    pi.sendMessage({
+      customType: 'research-config',
+      content: 'The interactive `/research-config` menu requires pi\'s terminal UI (TUI).\n\n' +
+        'In this host, use the headless diagnostics instead:\n' +
+        '- `/research-config health` — system health status\n' +
+        '- `/research-config knowledge-status` — knowledge store status\n\n' +
+        'Configuration is read from the environment and config files; set `PI_RESEARCH_*` ' +
+        'environment variables to change it.',
+      display: true,
+    });
     return;
   }
   await showInteractiveMenu(ctx, pi);
@@ -142,7 +165,19 @@ async function showInteractiveMenu(ctx: ExtensionContext, pi: ExtensionAPI): Pro
     {
       id: 'KNOWLEDGE_STORE_MODE',
       label: 'Knowledge Mode [project]',
-      description: 'Knowledge store scope — none (disabled), project (scope knowledge per this directory), or global (scope knowledge globally, shared across all projects).\n[project] means configured independently per directory.',
+      description: (() => {
+        const base = 'Knowledge store scope — none (disabled), project (scope knowledge per this directory), or global (scope knowledge globally, shared across all projects).\n[project] means configured independently per directory.';
+        // The configured mode is a lie when the host cannot run the store at all
+        // (optional embedding dep skipped at install, broken lancedb). Surface the
+        // package-level OFF right in the setting, so the menu never advertises a
+        // scope the store cannot honor and the repair path is named where the
+        // user is looking.
+        const availability = probeKnowledgeStoreAvailability();
+        if (!availability.available) {
+          return `${base}\n\n⚠ ${describeKnowledgeStoreUnavailability(availability)}\nThe store stays off — regardless of this setting — until that package is installed.`;
+        }
+        return base;
+      })(),
       currentValue: config.KNOWLEDGE_STORE_MODE,
       values: ['none', 'project', 'global'],
     },
@@ -485,6 +520,7 @@ async function showInteractiveMenu(ctx: ExtensionContext, pi: ExtensionAPI): Pro
         modelChangeConfirmed = !cancelled && await ctx.ui.confirm(
           'Change Embedding Model',
           `Switch the embedding model to ${config.EMBEDDING_MODEL.split('/').pop()}? This permanently clears the current knowledge store.`,
+          { timeout: 60_000 },
         );
         if (modelChangeConfirmed) {
           logger.info(`[research-config] Embedding model changed from ${initialConfig.EMBEDDING_MODEL} to ${config.EMBEDDING_MODEL}. Clearing knowledge store.`);
@@ -537,7 +573,7 @@ async function showInteractiveMenu(ctx: ExtensionContext, pi: ExtensionAPI): Pro
             await showKnowledgeStatusAction(ctx, pi);
             break;
           case 'knowledge_clear_global': {
-            const confirmed = await ctx.ui.confirm('Clear User Store', 'Permanently delete all user-scoped store data?');
+            const confirmed = await ctx.ui.confirm('Clear User Store', 'Permanently delete all user-scoped store data?', { timeout: 60_000 });
             if (confirmed) {
               try {
                 const service = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, ctx, container);
@@ -550,7 +586,7 @@ async function showInteractiveMenu(ctx: ExtensionContext, pi: ExtensionAPI): Pro
             break;
           }
           case 'knowledge_clear_local': {
-            const confirmed = await ctx.ui.confirm('Clear Project Store', 'Permanently delete all project-scoped store data?');
+            const confirmed = await ctx.ui.confirm('Clear Project Store', 'Permanently delete all project-scoped store data?', { timeout: 60_000 });
             if (confirmed) {
               try {
                 const service = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, ctx, container);
@@ -564,13 +600,13 @@ async function showInteractiveMenu(ctx: ExtensionContext, pi: ExtensionAPI): Pro
           }
           case 'metrics':
             await showMetricsAction(ctx, pi);
-            if (await ctx.ui.confirm('Session Metrics', 'Reset all session counters?')) {
+            if (await ctx.ui.confirm('Session Metrics', 'Reset all session counters?', { timeout: 60_000 })) {
               metrics.clearSession();
               ctx.ui.notify('Session metrics reset.', 'info');
             }
             break;
           case 'logs_clear': {
-            const confirmed = await ctx.ui.confirm('Clear Logs', 'Delete the main diagnostic log file and all archived rotation files?');
+            const confirmed = await ctx.ui.confirm('Clear Logs', 'Delete the main diagnostic log file and all archived rotation files?', { timeout: 60_000 });
             if (confirmed) {
               logger.clear();
               ctx.ui.notify('Diagnostic logs cleared.', 'info');
@@ -691,6 +727,7 @@ async function uninstallSkillAction(ctx: ExtensionContext, pi: ExtensionAPI): Pr
   const confirmed = await ctx.ui.confirm(
     'Uninstall Skill',
     `Remove the pi-research skill from ${names}? (Only symlinks this extension created are removed.)`,
+    { timeout: 60_000 },
   );
   if (!confirmed) return;
   try {

package/src/tools/research-knowledge-search.ts

@@ -840,6 +840,9 @@ function missResult(reason: string): AgentToolResult<unknown> {
     case 'store_disabled':
       text = 'No results found (knowledge store is disabled in settings). Live research can get the info.';
       break;
+    case 'store_packages_missing':
+      text = 'No results found (knowledge store is off — its required packages are not installed on this host, so it stays off regardless of settings). Live research can get the info.';
+      break;
     case 'store_not_ready':
       text = 'No results found (knowledge store is initializing). Live research can get the info.';
       break;
@@ -951,10 +954,16 @@ export function createResearchKnowledgeSearchTool(iface?: ConfigInterface): Tool
         // or null when the store is genuinely disabled/failed.
         const store = await storeService.getStore();
         if (!store) {
-          // null = disabled (mode still 'none') or a genuine init failure — distinguish for the
-          // user-facing message so a real failure doesn't read as "disabled in settings".
-          const lifecycle = (storeService as any).lifecycle;
-          return missResult(lifecycle === 'disabled' ? 'store_disabled' : 'store_not_ready');
+          // null = disabled (a settings choice OR a host whose store packages are
+          // missing) or a genuine init failure — distinguish all three in the
+          // user-facing message. A 'native' disable must not read as "disabled in
+          // settings": the repair is installing the package, not a Knowledge Mode
+          // toggle that cannot help (same truth-telling as knowledge-config, the
+          // /research-config menu, and the healthcheck).
+          const disabledReason = storeService.getDisabledReason();
+          if (disabledReason === 'native') return missResult('store_packages_missing');
+          if (disabledReason === 'mode') return missResult('store_disabled');
+          return missResult('store_not_ready');
         }
 
         const count = await store.count();