@lincoln504/pi-research 1.4.1 → 1.5.0

package/src/infrastructure/browser/thread-worker.ts

@@ -23,7 +23,7 @@ import {
 import {
   setWorkerId,
   initBrowser,
-  getContext,
+  acquireTaskContext,
   resetBrowser,
   cleanupBrowser,
   taskStarted,
@@ -107,23 +107,28 @@ async function runTask(data: TaskData | undefined): Promise<TaskResult> {
   // Registered around the whole browser-touching lifetime of this task so
   // resetBrowser() (below) can tell whether a sibling task dispatched to this
   // same worker process (WORKER_CONCURRENCY > 1) is still using the shared
-  // browser/context before tearing it down.
+  // browser before tearing it down.
   taskStarted();
+  // Task-owned, not module-owned: acquireTaskContext() below hands back a
+  // FRESH BrowserContext for this task alone, closed in the finally below —
+  // never shared with a concurrent or later sibling task.
+  let taskContext: any;
   try {
     await initBrowser();
     const initMs = Date.now() - startTime;
+    taskContext = await acquireTaskContext();
 
     let result: any;
     if (type === 'search') {
       if (!query) throw new Error('Search task requires a query');
-      const searchResult = await executeSearchTask(getContext(), query, abortController.signal);
+      const searchResult = await executeSearchTask(taskContext, query, abortController.signal);
       result = { results: searchResult.results, duration: Date.now() - startTime, jitter: searchResult.jitter };
     } else if (type === 'scrape') {
       if (!url) throw new Error('Scrape task requires a URL');
-      const scrapeResult = await executeScrapeTask(getContext(), url, abortController.signal);
+      const scrapeResult = await executeScrapeTask(taskContext, url, abortController.signal);
       result = { ...scrapeResult, duration: Date.now() - startTime };
     } else if (type === 'healthcheck') {
-      const healthResult = await executeHealthCheck(getContext(), initMs, abortController.signal);
+      const healthResult = await executeHealthCheck(taskContext, initMs, abortController.signal);
       result = { ...healthResult, duration: Date.now() - startTime };
     } else {
       result = { error: 'Unknown task type', duration: Date.now() - startTime };
@@ -156,6 +161,9 @@ async function runTask(data: TaskData | undefined): Promise<TaskResult> {
     };
   } finally {
     if (taskTimer !== undefined) clearTimeout(taskTimer);
+    if (taskContext) {
+      Promise.resolve(taskContext.close()).catch(() => { /* task is already done; a close failure here is not actionable */ });
+    }
     taskFinished();
   }
 }

package/src/infrastructure/embedding/embedding-server.ts

@@ -80,6 +80,11 @@ class SerialQueue {
   private readonly hardDeadlineMs: number;
   private readonly onPoison?: (err: Error) => void;
   private poisoned: Error | null = null;
+  // The currently-executing task's real work promise (not the soft-timeout race) —
+  // set only while a native call is actually in flight. Lets a caller that needs
+  // to dispose the underlying pipeline (EmbeddingServer.shutdown) wait for true
+  // idleness instead of racing disposal against a still-running inference.
+  private currentWork: Promise<unknown> | null = null;
 
   constructor(
     maxDepth = 200,
@@ -132,6 +137,7 @@ class SerialQueue {
         let hardTimer: ReturnType<typeof setTimeout> | undefined;
         try {
           work = fn();
+          this.currentWork = work;
           // Arm the hard-deadline watchdog from the moment the real GPU work starts, so
           // the cap is an honest wall-clock deadline measured from work-start rather than
           // (soft-timeout + hardDeadlineMs). A single inference still unsettled after
@@ -162,6 +168,7 @@ class SerialQueue {
           } finally {
             settled = true;
             if (hardTimer !== undefined) clearTimeout(hardTimer);
+            if (this.currentWork === work) this.currentWork = null;
           }
         }
       };
@@ -170,6 +177,43 @@ class SerialQueue {
     });
   }
 
+  /**
+   * Resolve once no native call is in flight AND no queued task is still due
+   * to run. Used by EmbeddingServer.shutdown() to wait out a still-running
+   * inference before disposing the pipeline — without this, disposal could
+   * run concurrently with a legitimately slow (not hung) embed/embedMany call
+   * on the same re-entrant-locked GPU context, the exact native-segfault
+   * class this queue exists to prevent. No timeout: a promise that truly
+   * never settles is already an accepted, pre-existing leak-to-process-exit
+   * scenario elsewhere in this class (see the poison path's
+   * disposeEmbedder=false handling) — better to leak than risk concurrent
+   * native access.
+   *
+   * Checking only `currentWork` is NOT enough with a second task already
+   * queued: a task's `run()` clears `currentWork` to null in its `finally`
+   * (once its native call has settled), and `pump()` only claims it again for
+   * the next task on a LATER microtask tick, once `run()` for the current
+   * task has fully unwound and `await entry.run()` resumes. If a caller's own
+   * check happens to land in that gap, `currentWork` reads as null even
+   * though a second native call is about to start (or a queue in the process
+   * of unwinding to its next task hasn't reached it yet) — exactly the window
+   * disposal must not run in. `running` stays true for the queue's entire
+   * drain, across every task in `tasks`, not just the current one, so it has
+   * no equivalent gap and closes this one.
+   */
+  async waitForIdle(): Promise<void> {
+    while (this.currentWork || this.running) {
+      if (this.currentWork) {
+        await this.currentWork.catch(() => {});
+      } else {
+        // Nothing executing right now, but pump() is still mid-drain (about
+        // to claim the next queued task, or unwinding) — yield a microtask
+        // so it can advance instead of reporting idle into this exact gap.
+        await Promise.resolve();
+      }
+    }
+  }
+
   private poison(err: Error): void {
     if (this.poisoned) return;
     this.poisoned = err;
@@ -414,6 +458,16 @@ export class EmbeddingServer implements IEmbedder {
     // is enough for another process to re-elect; the leaked native resources die with
     // eventual process exit.
     if (disposeEmbedder) {
+      // Wait for the SerialQueue to go idle BEFORE disposing. This shutdown() path
+      // also runs on ordinary leadership loss — not just the poison path above — and
+      // a legitimately slow (not hung) embed/embedMany can still be mid-flight on
+      // this same GPU context. Embedder.dispose()'s own drainActiveEmbeddings only
+      // waits 5s, an order of magnitude under the queue's own 120s soft timeout, so
+      // without this a normal large batch could have its pipeline disposed out from
+      // under it — the same concurrent-native-call hazard the queue exists to
+      // prevent, just triggered by disposal racing a live call instead of two calls
+      // racing each other.
+      await this.queue.waitForIdle();
       await this.embedder.dispose().catch((err) => {
         if (this.isShuttingDown) {
           logger.debug('[EmbeddingServer] Error disposing embedder during shutdown (expected):', err);

package/src/knowledge/embedder-init.ts

@@ -417,7 +417,8 @@ export async function handleWebGPUWarmupError(
   gpuLockHeld: boolean,
   model: string,
   initializationTimeoutMs: number,
-  useCache: boolean
+  useCache: boolean,
+  poolingMode: 'mean' | 'cls' | 'last_token' = 'mean'
 ): Promise<{ success: boolean; pipeline?: FeatureExtractionPipeline; error?: Error; dummy?: any }> {
   markWebGpuFallback();
 
@@ -451,7 +452,7 @@ export async function handleWebGPUWarmupError(
     logger.info('[embedder] CPU pipeline loaded successfully');
     
     // Warmup CPU pipeline
-    const { dummy, success, error: cpuWarmupErr } = await warmupPipeline(cpuPipeline, 'mean', useCache);
+    const { dummy, success, error: cpuWarmupErr } = await warmupPipeline(cpuPipeline, poolingMode, useCache);
     if (!success || cpuWarmupErr) {
       logger.error('[embedder] CPU fallback pipeline warmup failed:', cpuWarmupErr);
       if (cpuPipeline) {

package/src/knowledge/embedder.ts

@@ -331,7 +331,8 @@ export class Embedder {
             this.gpuLockHeld,
             this.model,
             this.initializationTimeoutMs,
-            this.useCache
+            this.useCache,
+            this.poolingMode
           );
           if (!result.success) {
             throw result.error;
@@ -404,9 +405,16 @@ export class Embedder {
       // the VECTOR — the full text is still stored verbatim and covered by FTS — but the
       // truncation is otherwise silent, so surface it: a spike here means chunks are
       // arriving larger than the embed cap and vector recall on their tail is degraded.
+      let keep = maxChars;
+      // slice() counts UTF-16 code units, so the cut can land between the halves of
+      // a surrogate pair (emoji, rare CJK) — same guard as truncateWithMarker
+      // (utils/text-utils.ts): a lone high surrogate is invalid text some
+      // tokenizers/providers choke on. Back off one unit to restore well-formed text.
+      const lastKeptUnit = text.charCodeAt(keep - 1);
+      if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
       metrics.increment('embedder_truncations_total', 1, { model: this.model });
-      logger.debug(`[embedder] Truncated input for embedding: ${text.length} -> ${maxChars} chars (model=${this.model})`);
-      return text.slice(0, maxChars);
+      logger.debug(`[embedder] Truncated input for embedding: ${text.length} -> ${keep} chars (model=${this.model})`);
+      return text.slice(0, keep);
     }
     return text;
   }

package/src/knowledge/index.ts

@@ -1,4 +1,5 @@
 import * as fs from 'node:fs';
+import * as path from 'node:path';
 import { KnowledgeStore } from './store.ts';
 import { WriterQueue } from './writer-queue.ts';
 import { Chunker } from './chunker.ts';
@@ -108,19 +109,44 @@ export async function createKnowledgeStoreComponents(
 }
 
 /**
- * Force delete the entire knowledge store directory.
- * Used for system-level clearing or recovery from corruption.
+ * Force delete the ACTIVE table's own directory, to recover from a corrupted
+ * LanceDB manifest. Scoped to that one table, not the whole dbDir: dbDir is a
+ * single unified store shared by every workspace and the global scope (rows
+ * are scoped by a `workspace` column, not by separate directories), and it
+ * also holds `_backup_*`/`_migration_*` sibling directories — the deliberate,
+ * user-recoverable safety net the 'backup' migration strategy creates. A
+ * directory-wide wipe here previously destroyed every other project's data
+ * and every backup the instant ANY workspace's session hit a corrupted
+ * manifest, on the one code path whose entire purpose is recovering from
+ * exactly that kind of damage.
  */
 export async function forceDeleteKnowledgeStore(config?: Config, workspace?: string): Promise<void> {
   const dbDir = getDbDir(config, workspace);
-  if (fs.existsSync(dbDir)) {
-    try {
-      fs.rmSync(dbDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
-      fs.mkdirSync(dbDir, { recursive: true });
-      logger.info(`[knowledge] Knowledge store at ${dbDir} forcefully deleted and recreated.`);
-    } catch (err) {
-      logger.error(`[knowledge] Failed to forcefully delete directory ${dbDir}:`, err);
-      throw err;
+  if (!fs.existsSync(dbDir)) return;
+
+  // Mirrors KnowledgeStore's own manifest read (store.ts): the active table
+  // name defaults to 'knowledge' but can point elsewhere after a persisted
+  // rename failure. A missing or unreadable manifest is not itself a reason to
+  // widen the blast radius, so it falls back to the default rather than
+  // escalating to a dbDir-wide delete.
+  let tableName = 'knowledge';
+  const manifestPath = path.join(dbDir, 'store-manifest.json');
+  try {
+    if (fs.existsSync(manifestPath)) {
+      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as { activeTableName?: string };
+      if (manifest.activeTableName) tableName = manifest.activeTableName;
     }
+  } catch (err) {
+    logger.warn(`[knowledge] Failed to read manifest before forced delete, defaulting to table '${tableName}':`, err);
+  }
+
+  const tableDir = path.join(dbDir, `${tableName}.lance`);
+  if (!fs.existsSync(tableDir)) return;
+  try {
+    fs.rmSync(tableDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
+    logger.info(`[knowledge] Corrupted table directory ${tableDir} forcefully deleted; will be recreated on next init.`);
+  } catch (err) {
+    logger.error(`[knowledge] Failed to forcefully delete table directory ${tableDir}:`, err);
+    throw err;
   }
 }

package/src/knowledge/store.ts

@@ -502,12 +502,92 @@ export class KnowledgeStore implements IKnowledgeStore {
         // If canonical name was already something else and it still exists, 
         // we might need to handle it. But we just dropped it above.
         await fsPromises.rename(tempDir, canonicalDir);
-        
+
         this.tableName = canonicalName;
-        await this.saveManifest();
-        
-        // Reopen the canonical table so this.table reflects the renamed path
-        this.table = await this.db.openTable(this.tableName);
+        if (await this.saveManifest()) {
+          // Reopen the canonical table so this.table reflects the renamed path
+          this.table = await this.db.openTable(this.tableName);
+        } else {
+          // The directory rename to the canonical name already happened on disk,
+          // but the manifest still says tempTableName (persisted successfully
+          // above, before the old table was dropped). Left as-is, the next
+          // process start would read that stale name, find no such directory
+          // (renamed away), and silently CREATE A NEW EMPTY TABLE under it —
+          // stranding every re-embedded document in the now-unreferenced
+          // canonical directory with no recovery path (pruneOrphanedMigrationDirs
+          // doesn't recognize a bare 'knowledge.lance' as orphaned migration
+          // material). Revert the rename so the manifest (still correctly
+          // pointing at tempTableName, from the pre-drop save above) and the
+          // on-disk layout agree again — the data stays fully intact and
+          // accessible under its temp name either way. Deliberately NOT thrown:
+          // the migration itself fully succeeded and the in-memory handle is
+          // fine, so surfacing this as a migration failure would trip the
+          // caller's re-embed-failed fallback to the 'backup' strategy, which
+          // would needlessly rename this perfectly good, fully re-embedded data
+          // out of the way and start a fresh empty table over one manifest
+          // write hiccup.
+          logger.error('[store] Failed to persist canonical table name after rename — reverting to keep manifest and disk in sync.');
+          try {
+            await fsPromises.rename(canonicalDir, tempDir);
+            this.tableName = tempTableName;
+            this.table = await this.db.openTable(this.tableName);
+            logger.info(`[store] Migrated data remains safe and accessible as ${tempTableName}; canonical rename deferred to a future successful manifest save.`);
+          } catch (revertErr) {
+            // The revert itself failed too. fs.rename is atomic on a single
+            // filesystem — a failed rename never leaves data half-moved — so
+            // the data is still fully at canonicalDir, not tempDir. The outer
+            // catch below was written for a DIFFERENT case (the first rename
+            // never happening at all) and would wrongly point the manifest at
+            // tempTableName, whose directory no longer exists on disk. Instead,
+            // try to persist a manifest against the name that actually matches
+            // disk right now, so the mismatch this whole branch exists to avoid
+            // doesn't reappear one level deeper.
+            logger.error(`[store] Revert rename also failed; data remains on disk at ${canonicalDir}: ${revertErr}`);
+            // Everything from here on is wrapped in its OWN try/catch and never
+            // rethrows: a third compounding failure (e.g. openTable() itself
+            // throwing, on the same underlying disk fault that already broke
+            // two renames) must not escape to the outer catch (renameErr)
+            // either — that one assumes data is still at tempDir, which by
+            // this point is doubly false.
+            // The manifest repair is deliberately NOT gated behind the table
+            // reopen: saveManifest depends only on writeFile/rename, not on an
+            // open table, so a reopen-only failure (a LanceDB-level fault on an
+            // otherwise-writable filesystem) must not skip a repair the code can
+            // still make — a skipped repair leaves the persisted manifest
+            // pointing at the nonexistent temp name, which is the next-restart
+            // empty-table hazard this whole branch exists to prevent.
+            this.tableName = canonicalName;
+            let reopened = false;
+            try {
+              this.table = await this.db.openTable(this.tableName);
+              reopened = true;
+            } catch (openErr) {
+              // A third compounding failure: even reopening the table at its
+              // actual on-disk location failed. This process has no working
+              // table handle for it, but the data itself should still be
+              // physically intact at canonicalDir (only the rename and its
+              // revert have run, and neither drops data — they only move a
+              // directory or fail outright). The manifest repair below still
+              // proceeds, so a future restart can find the data even though
+              // this process cannot serve it.
+              logger.error(`[store] CRITICAL: could not reopen the table at '${canonicalName}' after a failed rename revert (${openErr}). Data should still be intact on disk at ${canonicalDir}, but this process has no working table handle for it. Attempting the manifest repair anyway so a restart can find the data; verify ${canonicalDir} exists if problems persist.`);
+            }
+            try {
+              if (await this.saveManifest()) {
+                logger.warn(`[store] Recovered: manifest now points at '${canonicalName}', matching the data's actual on-disk location${reopened ? '' : ' (this process still has no working table handle — restart to serve it)'}.`);
+              } else {
+                // Both the revert and this recovery manifest save failed. The
+                // on-disk manifest (last persisted pointing at tempTableName,
+                // before the old table was dropped) now disagrees with disk.
+                // Fail loud rather than let a future restart silently create
+                // an empty table under the stale name.
+                logger.error(`[store] CRITICAL: could not persist a manifest matching disk after a failed rename revert. Data is intact on disk at ${canonicalDir} (table '${canonicalName}'), but the persisted manifest is stale and points at '${tempTableName}', which no longer exists. If this process exits before a future successful manifest save, the next start will not find the data automatically — manually repair the manifest to point at '${canonicalName}' before restarting.`);
+              }
+            } catch (manifestErr) {
+              logger.error(`[store] CRITICAL: manifest repair threw after a failed rename revert (${manifestErr}). Data is intact on disk at ${canonicalDir} (table '${canonicalName}'), but the persisted manifest is stale and points at '${tempTableName}'. Manually repair the manifest to point at '${canonicalName}' before restarting.`);
+            }
+          }
+        }
       } catch (renameErr) {
         // If rename fails (cross-device, permissions, etc.) we keep the temp name
         // but CRITICALLY we now persist it in the manifest so future sessions

package/src/observers/research-observer-impl.ts

@@ -43,6 +43,14 @@ export interface ObserverContext {
    */
   renderImmediate?: () => void;
   researchComplexity: number;
+  /**
+   * Whether prompt caching is active for the run's resolved model (see
+   * isPromptCachingActiveForModel). With caching active the effective
+   * per-researcher scrape-batch maximum is one higher, and progress units
+   * must be computed relative to that same effective count the tracker
+   * enforces — not the base configured value.
+   */
+  cachingActive?: boolean;
 }
 
 export interface ObserverState {
@@ -94,7 +102,7 @@ export function createResearchObserver(
         activateSlice(panelState, state.quickSliceLabel);
         updateSliceStatus(panelState, state.quickSliceLabel, 'starting...', debouncedRefresh);
 
-        const units = getUnitsPerResearcher();
+        const units = getUnitsPerResearcher(undefined, ctx.cachingActive);
         panelState.progress = { expected: units, made: 0 };
       }
       // Quick research: researcher is running → steering is acceptable
@@ -152,7 +160,7 @@ export function createResearchObserver(
     onPlanningSuccess: (plan) => {
       updateSliceStatus(panelState, 'coord', 'ready', debouncedRefresh);
       completeSlice(panelState, 'coord');
-      const unitsPerResearcher = getUnitsPerResearcher();
+      const unitsPerResearcher = getUnitsPerResearcher(undefined, ctx.cachingActive);
       const count = plan.researchers?.length || 0;
       const units = (count * unitsPerResearcher) + LEAD_EVAL_UNITS;
       panelState.progress = { expected: units, made: 0 };
@@ -319,7 +327,7 @@ export function createResearchObserver(
 
     onResearcherProgress: (id, status, tokens, cost) => {
       const sliceId = id === 'quick' ? state.quickSliceLabel : id;
-      const unitsPerResearcher = getUnitsPerResearcher();
+      const unitsPerResearcher = getUnitsPerResearcher(undefined, ctx.cachingActive);
       
       if (status !== undefined) {
         if (status.startsWith('done:')) {
@@ -350,7 +358,7 @@ export function createResearchObserver(
     onResearcherComplete: (id, _report) => {
       const sliceId = id === 'quick' ? state.quickSliceLabel : id;
       if (panelState.progress) {
-        const unitsPerResearcher = getUnitsPerResearcher();
+        const unitsPerResearcher = getUnitsPerResearcher(undefined, ctx.cachingActive);
         const current = progressCredits.get(id) ?? 0;
         const remaining = unitsPerResearcher - current;
         if (remaining > 0) {
@@ -365,7 +373,7 @@ export function createResearchObserver(
     onResearcherFailure: (id, _error) => {
       const sliceId = id === 'quick' ? state.quickSliceLabel : id;
       if (panelState.progress) {
-        const unitsPerResearcher = getUnitsPerResearcher();
+        const unitsPerResearcher = getUnitsPerResearcher(undefined, ctx.cachingActive);
         const current = progressCredits.get(id) ?? 0;
         const remaining = unitsPerResearcher - current;
         if (remaining > 0) {
@@ -430,7 +438,7 @@ export function createResearchObserver(
       } else {
         // Delegation: prepare for new round's search → researchers → steering acceptable
         if (plan?.researchers && plan.researchers.length > 0 && panelState.progress) {
-          const unitsPerResearcher = getUnitsPerResearcher();
+          const unitsPerResearcher = getUnitsPerResearcher(undefined, ctx.cachingActive);
           panelState.progress.expected += (plan.researchers.length * unitsPerResearcher) + LEAD_EVAL_UNITS;
         }
         panelState.steeringAcceptable = true;

package/src/orchestration/deep-research-orchestrator.ts

@@ -517,7 +517,9 @@ export class DeepResearchOrchestrator {
         // Guarded so a one-round budget cannot announce the same round twice.
         if (this.currentRound > lastAnnouncedRound) {
           observer?.onRoundStart?.(this.currentRound);
-          lastAnnouncedRound = this.currentRound;
+          // No assignment to lastAnnouncedRound here: this is the last place in the
+          // function that reads it — final synthesis proceeds unconditionally right
+          // after, nothing downstream checks it again.
         }
         // The round this synthesis actually happens at, not the budget. They coincide on
         // the round-cap exit but not on the wait-exhaustion one, which breaks out early —

package/src/orchestration/quick-research-orchestrator.ts

@@ -19,6 +19,7 @@ import { getConfig, type Config } from '../config.ts';
 import { createResearcherSession } from './researcher.ts';
 import { ensureAssistantResponse, parseCitations } from '../utils/text-utils.ts';
 import { getMaxScrapeBatches, resolveExcludedTools } from '../constants.ts';
+import { resolveResearchModel, isPromptCachingActiveForModel } from '../core/llm/research-model-resolver.ts';
 import type { ResearchObserver } from '../core/interfaces/observer-interfaces.ts';
 import { HeadlessObserver, makeSafeObserver, type HeadlessObserverOptions } from './headless-observer.ts';
 import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
@@ -133,7 +134,15 @@ export class QuickResearchOrchestrator {
         }
 
         const researcherPromptTemplate = loadPrompt('researcher');
-        const maxScrapeBatches = getMaxScrapeBatches(this.config);
+        // Resolve the model here (same deterministic inputs createResearcherSession
+        // below will resolve again) purely to learn whether prompt caching will be
+        // active, so the batch count told to the model in the prompt below matches
+        // the limit ToolUsageTracker actually enforces — resolveResearchModel is a
+        // pure, cheap lookup over already-loaded registries, safe to call twice.
+        const cachingActiveForBudget = isPromptCachingActiveForModel(
+          resolveResearchModel({ modelRegistry: ctx.modelRegistry, config: this.config, hostModel: model, cwd: ctx.cwd })
+        );
+        const maxScrapeBatches = getMaxScrapeBatches(this.config, cachingActiveForBudget);
         const maxScrapeBatchesDisplay = maxScrapeBatches > 99 ? 'unlimited' : maxScrapeBatches.toString();
 
         const quickEvidenceSection =
@@ -162,19 +171,29 @@ export class QuickResearchOrchestrator {
         }
         evidenceLines.push(quickEvidenceSection);
 
+        // The system prompt carries only run-invariant instructions; the goal,
+        // knowledge-store results, evidence, and steering go in the initial USER
+        // message below. A byte-identical system prompt + tool set is the cacheable
+        // prefix providers key on, so back-to-back quick runs (and every follow-up
+        // turn within this session) reuse it via prompt-cache reads.
         const prompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
-            // Function replacers: insert `$`-bearing dynamic content (scraped evidence,
-            // store descriptions, the user query) literally, not as a substitution pattern.
-            .replace('{{goal}}', () => query + steeringSection)
-            .replace('{{store_section}}', () => storeSection)
-            .replace('{{evidence_section}}', () => evidenceLines.join('\n\n'))
-            .replace('{{coordination_section}}', '')
             .replace('{{extra_tool_guidelines}}', '- `search`: Perform broad web searches (Round 1 only).')
             // Quick research has no research lead and no next round — see
             // RESEARCHER_DIGEST_SECTION. Its report IS the deliverable.
-            .replace('{{digest_section}}', '');
+            .replace('{{digest_section}}', '')
+            .trim();
 
-        logger.debug(`[QuickOrchestrator] System Prompt:\n${prompt}`);
+        // `$`-bearing dynamic content (the user query, store descriptions, steering)
+        // is plain string concatenation here — no String.replace substitution risk.
+        const userMessage = [
+          `Goal: ${query}`,
+          storeSection.trim(),
+          evidenceLines.join('\n\n').trim(),
+          steeringSection.trim(),
+          'Perform your research and submit your full report now.',
+        ].filter(Boolean).join('\n\n');
+
+        logger.debug(`[QuickOrchestrator] System Prompt:\n${prompt}\n\nUser Message:\n${userMessage}`);
 
         let lastSeenSearchCount = 0;
         const { session, resolvedModel } = await createResearcherSession({
@@ -287,7 +306,7 @@ export class QuickResearchOrchestrator {
           // whether the race resolves via session.prompt, timeout, or abort.
           let abortCleanup: (() => void) | undefined;
           try {
-            const promptPromise = session.prompt(query);
+            const promptPromise = session.prompt(userMessage);
             promptPromise.catch((err: Error) => logger.debug(`[QuickOrchestrator] Background session prompt rejection: ${err.message}`));
             await Promise.race([
               promptPromise,

package/src/orchestration/research-synthesis-service.ts

@@ -34,6 +34,22 @@ const NO_SOURCES_NOTE =
 const UNVERIFIED_LINKS_NOTE =
   '_None of the links above could be verified: no page fetch succeeded this session and no citation could be matched to retrieved content. They may be inaccurate or invented — check each one before relying on it._';
 
+// Below this many verified sources, a completed report is WEAKLY grounded: it will
+// still read as a fully sourced document, but almost none of its claims can trace to
+// retrieved content — exactly the shape of a run where scrapes mostly failed (bot
+// blocks, resource starvation) or the topic had little real coverage, and the model
+// filled the gaps from its own weights. Observed in production: a depth-1 run whose
+// report cited ONE page shipped with no caveat and fabricated most of its specifics;
+// the calling agent had to detect the degradation itself. The note gives readers
+// (human or agent) the machine-visible signal that was missing.
+const WEAK_GROUNDING_MIN_SOURCES = 3;
+
+function weakGroundingNote(count: number): string {
+  return `_Grounding notice: only ${count} source${count === 1 ? '' : 's'} could be verified for this report. ` +
+    'Claims not attributable to the cited source' + (count === 1 ? '' : 's') +
+    ' above should be treated as unverified — cross-check anything load-bearing before relying on it._';
+}
+
 
 /**
  * Redact inline `https?://` tokens in synthesis prose whose URL was NOT retrieved
@@ -382,6 +398,16 @@ export class ResearchSynthesisService implements IService {
         // fabricated source can still reach the user; inline URLs also violate the
         // required format. Verified inline references (in `trustedUrls`) are kept.
         body = redactUnverifiedProseUrls(body, trustedUrls);
+        if (globalCitations.length < WEAK_GROUNDING_MIN_SOURCES) {
+          // The notice goes BEFORE the CITED LINKS header, never after it: anything
+          // trailing the last citation entry is absorbed into that entry's
+          // Description by parseCitations, and quick mode re-parses this exact
+          // string to build knowledge-store entries — a notice placed after the
+          // list polluted the stored description of the final source on precisely
+          // the degraded runs the notice exists to flag.
+          logger.warn(`[ResearchSynthesisService] Report is weakly grounded: only ${globalCitations.length} verified source(s) — appending grounding notice.`);
+          return `${body}\n\n${weakGroundingNote(globalCitations.length)}\n\n${verifiedLinksSection}`;
+        }
         return `${body}\n\n${verifiedLinksSection}`;
       }
       logger.warn('[ResearchSynthesisService] Synthesis missing CITED LINKS - appending verified version');
@@ -393,6 +419,12 @@ export class ResearchSynthesisService implements IService {
       const validIds = new Set(globalCitations.map((c) => c.id));
       let prose = rewriteCitationMarkers(synthesis.trim(), new Map(), (n) => !validIds.has(n));
       prose = redactUnverifiedProseUrls(prose, trustedUrls);
+      if (globalCitations.length < WEAK_GROUNDING_MIN_SOURCES) {
+        // Before the CITED LINKS header for the same parseCitations-absorption
+        // reason as the replace path above.
+        logger.warn(`[ResearchSynthesisService] Report is weakly grounded: only ${globalCitations.length} verified source(s) — appending grounding notice.`);
+        return `${prose}\n\n${weakGroundingNote(globalCitations.length)}\n\n${verifiedLinksSection}`;
+      }
       return `${prose}\n\n${verifiedLinksSection}`;
     }
 

package/src/orchestration/researcher-executor.ts

@@ -180,6 +180,20 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
   const researcherExecutionStartMs = Date.now();
   const deliveredSteeringIds = new Set<string>();
 
+  // The system prompt is deliberately invariant across researchers, rounds, AND
+  // retry attempts: everything per-researcher (goal, knowledge-store results,
+  // initial evidence, sibling coordination, steering) is delivered in the initial
+  // USER message instead. A byte-identical system prompt + tool set is the
+  // cacheable prefix providers key on, so every researcher after the first gets a
+  // prompt-cache read on that prefix rather than a fresh write. Built once, out
+  // of the retry loop, because nothing in it varies per attempt.
+  const researcherSystemPrompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
+    .replace('{{extra_tool_guidelines}}', '')
+    // Function replacer so `$`-bearing content is inserted literally, not as a
+    // String.replace substitution pattern.
+    .replace('{{digest_section}}', () => RESEARCHER_DIGEST_SECTION)
+    .trim();
+
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     // Launch boundary: never build a session for a run that is already cancelled
     // or fast-stopped. The abort sentinel keeps the wrapper's clean-cancel
@@ -221,30 +235,31 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
       }
     }
 
-    // Generate prompt for this attempt, incorporating ALL steering messages delivered so far
+    // Build this attempt's initial USER message: the researcher's assignment plus
+    // every per-researcher section that used to live in the system prompt (goal,
+    // knowledge-store results, initial evidence, sibling coordination), and ALL
+    // steering messages delivered so far. Rebuilt per attempt so a retry picks up
+    // steering that arrived between attempts; the system prompt above never changes.
     const allSteering = getSteeringMessages(sessionId);
     let steeringSection = '';
     if (allSteering.length > 0) {
-        steeringSection = '\n\n### ADDITIONAL USER GUIDANCE (Mandatory directional rules for your research)\n' +
+        steeringSection = '### ADDITIONAL USER GUIDANCE (Mandatory directional rules for your research)\n' +
             allSteering.map(m => {
                 deliveredSteeringIds.add(m.id);
                 return `- ${m.text}`;
             }).join('\n');
     }
 
-    const prompt = injectCurrentDate(researcherPromptTemplate, 'researcher')
-      // Function replacers so a `$`-bearing dynamic value (scraped evidence, store
-      // descriptions, or an LLM-authored goal containing $&, $$, $1, …) is inserted
-      // literally instead of being interpreted as a String.replace substitution pattern.
-      .replace('{{goal}}', () => researcherConfig.goal)
-      .replace('{{store_section}}', () => storeSection)
-      .replace('{{evidence_section}}', () => evidenceSection)
-      .replace('{{coordination_section}}', () => previousQueriesSection)
-      .replace('{{extra_tool_guidelines}}', '')
-      .replace('{{digest_section}}', () => RESEARCHER_DIGEST_SECTION)
-      .trim() + steeringSection;
+    const userMessage = [
+      `Topic: ${researcherConfig.name}\nGoal: ${researcherConfig.goal}`,
+      storeSection.trim(),
+      evidenceSection.trim(),
+      previousQueriesSection.trim(),
+      steeringSection,
+      'Perform your research and submit your full report now.',
+    ].filter(Boolean).join('\n\n');
 
-    logger.debug(`[ResearcherExecutor] Researcher ${id} attempt ${attempt} System Prompt:\n${prompt}`);
+    logger.debug(`[ResearcherExecutor] Researcher ${id} attempt ${attempt} System Prompt:\n${researcherSystemPrompt}\n\nUser Message:\n${userMessage}`);
 
     const workerExclude = ['search'];
     const mergedExclude = [...new Set([...workerExclude, ...(options.excludeTools || [])])];
@@ -275,7 +290,7 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
       cwd: ctx.cwd,
       ctxModel: model,
       modelRegistry: ctx.modelRegistry,
-      systemPrompt: prompt,
+      systemPrompt: researcherSystemPrompt,
       extensionCtx: ctx,
       excludeTools: mergedExclude,
       researcherId: id,
@@ -434,7 +449,7 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
 
       let abortCleanup: (() => void) | undefined;
       try {
-        const promptPromise = session.prompt(`Topic: ${researcherConfig.name}\nGoal: ${researcherConfig.goal}\n\nPerform your research and submit your full report now.`);
+        const promptPromise = session.prompt(userMessage);
         promptPromise.catch((err: Error) => logger.debug(`[ResearcherExecutor] Background session prompt rejection: ${err.message}`));
         await Promise.race([
           promptPromise,