@lincoln504/pi-research 1.5.3 → 1.6.1

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -6,7 +6,7 @@
 
 let workerId: string = '';
 
-import { appendFileSync } from 'node:fs';
+import { appendFileSync, promises as fsp } from 'node:fs';
 import { redactSecrets } from '../../utils/log-utils.ts';
 import { safeUnref } from '../../utils/safe-unref.ts';
 import { MAX_HTML_SIZE, MAX_PDF_SIZE } from '../../web-research/scraper-types.ts';
@@ -405,6 +405,156 @@ function seedSubresourceVerdict(validatedUrl: string): void {
   }
 }
 
+/**
+ * True when Firefox has discarded a response body it once held, so `response.body()`
+ * can never succeed for that response again. Distinct from every other body failure —
+ * a closed target, an aborted task — which must keep their original error.
+ */
+function isEvictedBodyError(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+
+/** Upper bound on the PDF re-fetch below. A 14MB PDF measured ~20s on a normal link. */
+const PDF_REFETCH_TIMEOUT_MS = 90_000;
+
+/**
+ * Re-request a PDF whose navigation body Firefox has already discarded.
+ *
+ * Uses the browser context's own request API rather than a bare fetch so the retry
+ * carries the same cookies, headers, proxy and TLS profile the navigation used — the
+ * reason this URL reached the stealth-browser fallback in the first place is usually
+ * that a plain fetch does not get served.
+ *
+ * SSRF: this is the one request in the scrape path that does NOT pass through
+ * `page.route` or fire `page.on('response')`, so neither the request-time DNS check
+ * nor the serverAddr rebinding backstop covers it. Three things stand in for them:
+ * the URL is re-validated here with the same DNS-aware check the route guard uses;
+ * it is the FINAL navigation URL, whose actually-connected IP the caller validated
+ * via `response.serverAddr()` seconds earlier; and redirects are refused outright, so
+ * the re-fetch cannot be walked to an unvalidated host. What remains is a TTL-0
+ * rebind landing inside the seconds between the navigation and this retry, on a host
+ * that already answered from a public address — strictly narrower than the
+ * subresource TOCTOU window the request-time check already accepts.
+ */
+async function refetchPdfBody(
+  page: any,
+  url: string,
+  validateForWorker: (u: string) => Promise<unknown>,
+  signal?: AbortSignal,
+): Promise<Buffer> {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error('Aborted');
+
+  // Raced against the task signal, not merely checked before starting. Playwright's
+  // request API takes no AbortSignal, and closing the page does not cancel it — so
+  // without this race a slow retry would hold the poolifier worker slot for up to
+  // PDF_REFETCH_TIMEOUT_MS past the deadline that was supposed to free it. The request
+  // itself is left to settle and be discarded; what matters is that the slot is not
+  // pinned waiting for it.
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS,
+  });
+  requestPromise.catch(() => {});
+  const response = await (signal
+    ? Promise.race([
+        requestPromise,
+        new Promise<never>((_, reject) => {
+          if (signal.aborted) return reject(new Error('Aborted'));
+          signal.addEventListener('abort', () => reject(new Error('Aborted')), { once: true });
+        }),
+      ])
+    : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  // A 3xx here means the server wants to send us somewhere maxRedirects:0 refused to
+  // follow — report it as the HTTP outcome it is rather than reading a redirect body.
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h: any) => String(h.name).toLowerCase() === 'content-length')?.value ?? ''), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+
+  const buffer: Buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile('DEBUG', `[Worker-${workerId}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+
+/**
+ * Finish a scrape from a Download the navigation turned into.
+ *
+ * A server that sends a PDF with `Content-Disposition: attachment` never produces a
+ * document for Playwright to inspect — Firefox hands the response to the download
+ * manager and `page.goto()` rejects with "Download is starting". The bytes have
+ * already arrived at that point, so the URL is recoverable; before this path existed
+ * every such source (vendor datasheets, standards-body PDFs, several journal hosts)
+ * was reported as an outright scrape failure.
+ *
+ * The SSRF guarantees are the ones the normal PDF branch relies on: the download's
+ * response passed through the same `page.on('response')` inspection, so settling
+ * `pendingAddrChecks` and re-checking the poison gate before returning bytes is
+ * equivalent here.
+ */
+async function readPdfDownload(
+  download: any,
+  url: string,
+  page: any,
+  pendingAddrChecks: Array<Promise<void>>,
+  getPoisonedError: () => Error | null,
+): Promise<{ contentType: string; bufferB64: string; jitter: number }> {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+
+  // Only PDFs are usable as binary content downstream; anything else (zip, csv, xlsx)
+  // is a routine unsupported-target outcome for this URL, not a fault.
+  const suggested = String(download.suggestedFilename?.() ?? '');
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf =
+    suggested.toLowerCase().endsWith('.pdf') || downloadUrl.toLowerCase().split('?')[0]!.endsWith('.pdf');
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+
+  const filePath = await download.path();
+  if (!filePath) throw new Error('Fetch blocked: download produced no readable file');
+
+  // Same 100MB policy cap the PDF branch enforces, checked from the file size so the
+  // bytes are never read into the worker at all when the file is over the limit.
+  //
+  // The size is measured on an OPEN HANDLE and the bytes are read back through that
+  // same handle, so the check and the read cannot land on two different files: a
+  // stat-by-path followed by a read-by-path is a time-of-check/time-of-use gap, and
+  // the download directory is a path the browser process is still writing to.
+  const handle = await fsp.open(filePath, 'r');
+  let buffer: Buffer;
+  try {
+    const { size } = await handle.stat();
+    if (size > MAX_PDF_SIZE) {
+      const sizeMB = Math.round(size / 1024 / 1024);
+      throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+    }
+    buffer = await handle.readFile();
+  } finally {
+    await handle.close().catch(() => {});
+  }
+
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+
+  // Delete before the page closes: the context's download directory is removed on
+  // context close, but a worker's context is reused across tasks, so an undeleted
+  // file would accumulate for the life of the worker.
+  await download.delete?.().catch(() => {});
+  await page.close();
+  logToDebugFile('DEBUG', `[Worker-${workerId}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: 'application/pdf', bufferB64: buffer.toString('base64'), jitter: 0 };
+}
+
 /**
  * Execute a scrape task
  */
@@ -446,6 +596,14 @@ export async function executeScrapeTask(
   // page.close() produces.
   let oversizedPdfError: Error | null = null;
 
+  // Set when the navigation turns into a download instead of a document. A server that
+  // sends the PDF with Content-Disposition: attachment makes Firefox hand the response
+  // to the download manager, and page.goto() then rejects with "Download is starting" —
+  // the PDF branch below was never reached and the document was lost even though the
+  // bytes had already arrived. Captured here so the goto catch can finish the scrape
+  // from the downloaded file instead of failing the URL.
+  let pendingDownload: any = null;
+
   try {
     logToDebugFile('DEBUG', `[Worker-${workerId}] Starting scrape for: ${url}`);
 
@@ -589,8 +747,32 @@ export async function executeScrapeTask(
       }
     });
 
+    // A navigation the server marks as an attachment never becomes a document: Firefox
+    // routes it to the download manager and goto() rejects. Record the Download object
+    // so the goto catch below can read the bytes that already arrived.
+    page.on('download', (d: any) => { pendingDownload ??= d; });
+
     // High-fidelity wait: try domcontentloaded first for speed
-    const response = await page.goto(url, { waitUntil: 'domcontentloaded' });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' });
+    } catch (navErr: unknown) {
+      // Only the download case is recoverable here; everything else (timeout, DNS,
+      // 'Target closed' from an abort/poison) must keep its original error.
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      // The 'download' event and the goto rejection are not ordered relative to each
+      // other, so give the event a brief moment to land before giving up on it.
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent('download', { timeout: 5000 }).catch(() => null),
+          new Promise(r => setTimeout(r, 5000)),
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
 
     // DNS-rebinding defense-in-depth. The page.route guard's validateForWorker resolves the
     // hostname independently of the browser's own connect-time resolution, so a TTL-0 rebind can
@@ -647,7 +829,18 @@ export async function executeScrapeTask(
       // decided this and be in the middle of closing the page — skip the (redundant,
       // about-to-fail) download attempt rather than race it.
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer: Buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        // Firefox drops large navigation response bodies from its juggler-side cache,
+        // after which response.body() can only ever fail. Measured against a 13.9MB
+        // TU Delft lecture PDF: the body is unavailable whether it is requested at
+        // headers-received time or after the load settles, so this is not a race that
+        // asking earlier can win — the bytes are simply no longer held for us.
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       // ...then the actual bytes: the header is advisory and absent on chunked
       // responses, so the byte count is the check that actually bounds memory.
       // Checked BEFORE toString('base64') doubles the allocation.

package/src/infrastructure/browser/thread-worker.mjs

@@ -1516,11 +1516,25 @@ function isPoolShutdownError(error) {
   // same transient swap the "being reset" string covers, not a hard fault.
   err.message.includes("Cannot execute a task on not started pool"));
 }
-var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
+function isBenignScrapeFailure(error) {
+  const msg = error instanceof Error ? error.message : String(error);
+  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
+  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
+  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
+  msg.startsWith("Unsupported download:") || // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+  BENIGN_NAVIGATION_FAILURE.test(msg) || // remote TLS/DNS/timeout — the URL could not be loaded
+  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
+  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
+  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
+  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
+}
+var BENIGN_NAVIGATION_FAILURE, DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({
   "src/infrastructure/browser/browser-error-utils.ts"() {
     "use strict";
     init_circuit_breaker();
+    BENIGN_NAVIGATION_FAILURE = /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
     DEFAULT_BREAKER_CONFIG = {
       // Raised from 3→8: parallel search bursts of 20 queries frequently produce
       // simultaneous timeouts on slow/CF-protected sites. A threshold of 3 caused
@@ -1565,17 +1579,6 @@ import * as dnsCb from "node:dns";
 import * as net from "node:net";
 import crypto from "node:crypto";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-function isBenignScrapeFailure(error) {
-  const msg = error instanceof Error ? error.message : String(error);
-  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
-  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
-  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
-  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
-  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
-  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
-  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
-}
 function getRandomUserAgent() {
   try {
     const index = crypto.randomInt(0, USER_AGENTS.length);
@@ -2014,7 +2017,7 @@ import { platform as platform2 } from "node:os";
 
 // src/infrastructure/browser/thread-worker-messaging.ts
 init_log_utils();
-import { appendFileSync as appendFileSync2 } from "node:fs";
+import { appendFileSync as appendFileSync2, promises as fsp } from "node:fs";
 
 // src/utils/safe-unref.ts
 function safeUnref(timer) {
@@ -2286,6 +2289,73 @@ function seedSubresourceVerdict(validatedUrl) {
   } catch {
   }
 }
+function isEvictedBodyError(err) {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+var PDF_REFETCH_TIMEOUT_MS = 9e4;
+async function refetchPdfBody(page, url, validateForWorker, signal) {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error("Aborted");
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS
+  });
+  requestPromise.catch(() => {
+  });
+  const response = await (signal ? Promise.race([
+    requestPromise,
+    new Promise((_, reject) => {
+      if (signal.aborted) return reject(new Error("Aborted"));
+      signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
+    })
+  ]) : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h) => String(h.name).toLowerCase() === "content-length")?.value ?? ""), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+  const buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+async function readPdfDownload(download, url, page, pendingAddrChecks, getPoisonedError) {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+  const suggested = String(download.suggestedFilename?.() ?? "");
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf = suggested.toLowerCase().endsWith(".pdf") || downloadUrl.toLowerCase().split("?")[0].endsWith(".pdf");
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+  const filePath = await download.path();
+  if (!filePath) throw new Error("Fetch blocked: download produced no readable file");
+  const handle = await fsp.open(filePath, "r");
+  let buffer;
+  try {
+    const { size } = await handle.stat();
+    if (size > MAX_PDF_SIZE) {
+      const sizeMB = Math.round(size / 1024 / 1024);
+      throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+    }
+    buffer = await handle.readFile();
+  } finally {
+    await handle.close().catch(() => {
+    });
+  }
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+  await download.delete?.().catch(() => {
+  });
+  await page.close();
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: "application/pdf", bufferB64: buffer.toString("base64"), jitter: 0 };
+}
 async function executeScrapeTask(_context, url, signal) {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
@@ -2302,6 +2372,7 @@ async function executeScrapeTask(_context, url, signal) {
   let poisonedError = null;
   const pendingAddrChecks = [];
   let oversizedPdfError = null;
+  let pendingDownload = null;
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
     const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2368,7 +2439,25 @@ async function executeScrapeTask(_context, url, signal) {
       } catch {
       }
     });
-    const response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    page.on("download", (d) => {
+      pendingDownload ??= d;
+    });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    } catch (navErr) {
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent("download", { timeout: 5e3 }).catch(() => null),
+          new Promise((r) => setTimeout(r, 5e3))
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
     const serverAddr = await response?.serverAddr().catch(() => null);
     if (serverAddr?.ipAddress) {
       const ip = serverAddr.ipAddress;
@@ -2386,7 +2475,13 @@ async function executeScrapeTask(_context, url, signal) {
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       if (buffer.byteLength > MAX_PDF_SIZE) {
         const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
@@ -2749,7 +2844,7 @@ function logToDebugFile3(level, ...args) {
         return String(arg);
       }).join(" "))
     };
-    import("node:fs/promises").then((fsp) => fsp.appendFile(logFile, `${JSON.stringify(entry)}
+    import("node:fs/promises").then((fsp2) => fsp2.appendFile(logFile, `${JSON.stringify(entry)}
 `)).catch(() => {
     });
   } catch {
@@ -2888,7 +2983,7 @@ async function initBrowser() {
   return initPromise;
 }
 async function acquireTaskContext() {
-  const contextPromise = browser.newContext();
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId;
   let contextRaceLost = false;

package/src/knowledge/store-operations.ts

@@ -13,6 +13,13 @@ import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
 import { StoreDocument } from '../core/interfaces/knowledge-interfaces.ts';
 
+/**
+ * What a single add attempt actually did. Returned rather than logged/counted here
+ * so the caller — which owns the retry loop — records exactly one outcome per
+ * logical addDocuments call.
+ */
+export type AddDocumentsOutcome = 'written' | 'empty' | 'ignored_closing';
+
 /**
  * Guard against a broken embedder silently writing corrupt vectors: a
  * wrong-count, empty, ragged, or NaN/Inf-poisoned result would otherwise land
@@ -69,12 +76,11 @@ export async function addDocumentsToStore(
   isClosing: () => boolean,
   workspace: string,
   isGlobal: boolean
-): Promise<void> {
-  if (docs.length === 0) return;
+): Promise<AddDocumentsOutcome> {
+  if (docs.length === 0) return 'empty';
   if (isClosing()) {
     logger.warn('[store] Ignoring addDocuments during close');
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
-    return;
+    return 'ignored_closing';
   }
 
   const startTime = Date.now();
@@ -98,14 +104,19 @@ export async function addDocumentsToStore(
     await table.add(data);
     const duration = Date.now() - startTime;
     metrics.observe('knowledge_store_add_documents_duration_ms', duration);
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'success' });
     metrics.increment('knowledge_store_chunks_added_total', docs.length);
     logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [workspace=${workspace}, global=${isGlobal}]`);
+    return 'written';
   } catch (err) {
     const duration = Date.now() - startTime;
     metrics.observe('knowledge_store_add_documents_duration_ms', duration, { status: 'error' });
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
-    logger.error('[store] Failed to add documents:', err);
+    // Attempt-level failure only. The caller retries embedder-unreachable and Lance
+    // commit-conflict failures and usually succeeds, so logging ERROR and counting a
+    // terminal outcome here reported lost writes that were not lost, and double-counted
+    // the batch in knowledge_store_add_documents_total (once error, once success).
+    // Outcome accounting belongs to KnowledgeStore.addDocuments, which is the only
+    // layer that knows whether the write was ultimately abandoned.
+    logger.debug(`[store] add attempt failed after ${duration}ms: ${err instanceof Error ? err.message : String(err)}`);
     throw err;
   }
 }

package/src/knowledge/store.ts

@@ -23,6 +23,7 @@ import { MigrationStrategy, MigrationResult } from './migration.ts';
 import { metrics } from '../utils/metrics.ts';
 import { createStoreTable, CURRENT_SCHEMA_VERSION } from './store-schema.ts';
 import { addDocumentsToStore, assertValidEmbeddingVectors, searchStore, findDocumentsByUrl, findRelevantUrls } from './store-operations.ts';
+import type { AddDocumentsOutcome } from './store-operations.ts';
 import { isEmbedderUnreachable } from './embedder-utils.ts';
 import { ServiceLifecycle } from '../core/service-registry.ts';
 import { ServiceNames } from '../core/service-interfaces.ts';
@@ -713,13 +714,14 @@ export class KnowledgeStore implements IKnowledgeStore {
       // Now check isClosing after registering
       if (this.isClosing) {
         logger.warn(`[store] Skipping addDocuments — store is closing`);
+        metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
         return;
       }
 
       let table = await this.getFreshTable();
       const workspace = this.getWorkspace();
 
-      await this.withEmbedderReconnect(async (embedder) => {
+      const outcome = await this.withEmbedderReconnect<AddDocumentsOutcome>(async (embedder) => {
         let retryCount = 0;
         const MAX_RETRIES = 5;
         const BASE_DELAY = 100;
@@ -727,7 +729,7 @@ export class KnowledgeStore implements IKnowledgeStore {
         while (retryCount <= MAX_RETRIES) {
           try {
             const isGlobal = this.options.knowledgeMode === 'global';
-          await addDocumentsToStore(
+            return await addDocumentsToStore(
               table, 
               docs, 
               embedder, 
@@ -735,7 +737,6 @@ export class KnowledgeStore implements IKnowledgeStore {
               workspace,
               isGlobal
             );
-            return;
           } catch (err) {
             // Detect commit conflicts typical of cross-process write contention
             if (isLanceCommitConflict(err)) {
@@ -758,7 +759,24 @@ export class KnowledgeStore implements IKnowledgeStore {
             throw err;
           }
         }
+        // Unreachable: the loop only exits via return or throw. Present so the
+        // outcome type stays non-optional for the accounting below.
+        throw new Error('[store] addDocuments retry loop exited without result');
       });
+
+      // Exactly one outcome per logical addDocuments call. Attempts that failed and
+      // were then recovered by the reconnect/conflict retries are not counted here
+      // and are not logged at ERROR — only a write that is actually abandoned is.
+      // An empty batch is a no-op and is not counted at all, as before.
+      if (outcome !== 'empty') {
+        metrics.increment('knowledge_store_add_documents_total', 1, {
+          status: outcome === 'ignored_closing' ? 'ignored_closing' : 'success',
+        });
+      }
+    } catch (err) {
+      metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
+      logger.error('[store] Failed to add documents:', err);
+      throw err;
     } finally {
       this.activeWrites.delete(writePromise);
       writeResolve!();

package/src/orchestration/deep-research-orchestrator.ts

@@ -9,7 +9,8 @@ import { type Model } from '@earendil-works/pi-ai';
 import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
 import { getSteeringMessages, consumeQueuedMessages, getActiveSteeringMessages, getFailedResearchers, getResearcherFailureReasons } from './session-state.ts';
-import { MAX_EXTRA_ROUNDS_WITH_STEERING, resolveExcludedTools } from '../constants.ts';
+import { MAX_EXTRA_ROUNDS_WITH_STEERING, MIN_SYNTHESIS_PROSE_CHARS, SYNTHESIS_FALLBACK_NOTICE, resolveExcludedTools } from '../constants.ts';
+import { synthesisProseBody, isAnalysisFreeSynthesis } from '../utils/text-utils.ts';
 import {
   getMaxRounds,
 } from '../core/planning-utils.ts';
@@ -594,6 +595,19 @@ export class DeepResearchOrchestrator {
 
       let result = finalReport.content || '';
 
+      // Unwrap FIRST, before anything judges the content. If the LLM returned the full
+      // JSON envelope as its response text and JSON parsing failed upstream,
+      // finalReport.content is the raw JSON string rather than the extracted markdown —
+      // and every check below would then be measuring the envelope instead of the report.
+      // An envelope wrapping an empty or title-only content read as a long, well-punctuated
+      // document, so neither guard fired and the unwrapped nothing went straight out.
+      try {
+        const parsed: unknown = JSON.parse(result);
+        if (parsed && typeof parsed === 'object' && typeof (parsed as Record<string, unknown>)['content'] === 'string') {
+          result = (parsed as Record<string, unknown>)['content'] as string;
+        }
+      } catch { /* result is not JSON — use as-is */ }
+
       if (!result.trim()) {
           if (hasReports) {
               // Content is empty but reports exist — build a fallback synthesis.
@@ -603,18 +617,30 @@ export class DeepResearchOrchestrator {
               // No reports, no recorded failures, and nothing to ship either.
               throw new Error('Research produced no report — no source material was collected.');
           }
+      } else if (hasReports && isAnalysisFreeSynthesis(result, MIN_SYNTHESIS_PROSE_CHARS)) {
+          // The synthesis LLM returned something, but nothing that constitutes analysis —
+          // twice in the run logs it was a lone title line. `!result.trim()` passes on
+          // that, and ensureCitedLinks then appends the rebuilt source list underneath, so
+          // a 131-character failure reached the user as a 6.8KB document with a heading,
+          // citations, and not one sentence between them, reported as a successful run.
+          //
+          // The response is deliberately ADDITIVE rather than a replacement. Whatever the
+          // model did write is kept, the researchers' own reports are appended below it,
+          // and a notice states plainly what happened. That matters more than the
+          // detector's precision: because a false positive can only ever ADD material,
+          // never discard it, the threshold can sit far away from real reports instead of
+          // hugging them out of fear of destroying a legitimate one. A missed detection
+          // costs a thin report; a false detection costs a few paragraphs of duplication.
+          const proseChars = synthesisProseBody(result).length;
+          logger.warn(`[DeepOrchestrator] LLM synthesis for ${researchId} carried no analysis (${proseChars} char(s) of prose, min ${MIN_SYNTHESIS_PROSE_CHARS}) — appending ${synthesisCheckService.getReportCount(researchId)} researcher report(s) below it`);
+          metrics.increment('research_synthesis_analysis_free_total', 1, { mode: 'deep' });
+          result = [
+              result.trim(),
+              SYNTHESIS_FALLBACK_NOTICE,
+              synthesisCheckService.buildFallbackSynthesis(researchId, this.currentRound),
+          ].join('\n\n');
       }
 
-      // Guard: if the LLM returned the full JSON envelope as its response text and JSON
-      // parsing failed upstream, finalReport.content may be the raw JSON string rather
-      // than the extracted markdown. Unwrap it here so docs never show the raw JSON.
-      try {
-        const parsed: unknown = JSON.parse(result);
-        if (parsed && typeof parsed === 'object' && typeof (parsed as Record<string, unknown>)['content'] === 'string') {
-          result = (parsed as Record<string, unknown>)['content'] as string;
-        }
-      } catch { /* result is not JSON — use as-is */ }
-
       // Final post-processing: Ensure CITED LINKS section is accurate and consistent
       const synthesisService = await getService<IResearchSynthesisService>(ServiceNames.RESEARCH_SYNTHESIS_SERVICE, ctx, container);
       result = synthesisService.ensureCitedLinks(researchId, result);