@lincoln504/pi-research 1.1.2 → 1.2.0

package/src/utils/http-body.ts

@@ -0,0 +1,176 @@
+/**
+ * Capped response-body reading.
+ *
+ * `response.json()` and `response.text()` buffer the WHOLE body into the main
+ * process before anything can inspect its size — the byte count is only knowable
+ * after the memory has already been committed. A hostile or merely broken upstream
+ * can therefore drive this process to an OOM kill with a single reply, and because
+ * a `Content-Length` header is advisory (absent on chunked encoding, and freely
+ * falsifiable) no pre-screen closes that hole. Streaming chunk-by-chunk and
+ * aborting the instant the cap is crossed is the only check that actually bounds
+ * the allocation.
+ *
+ * The scraper has read HTML and PDF this way for some time; these helpers exist so
+ * the JSON API clients share that one implementation rather than each growing a
+ * near-copy of it.
+ */
+
+/** Thrown by {@link readBodyCapped} when a streamed body crosses its byte cap. */
+export class BodyTooLargeError extends Error {
+  constructor(public readonly bytesRead: number, public readonly maxBytes: number) {
+    super(`Response body exceeded ${maxBytes} byte cap`);
+    this.name = 'BodyTooLargeError';
+  }
+}
+
+/**
+ * Default cap for a JSON API response.
+ *
+ * Every endpoint behind these clients answers in kilobytes for a normal query, so
+ * 32MB is orders of magnitude of headroom for legitimate traffic while still
+ * bounding a runaway body to something a Node heap absorbs without dying. Bulk
+ * endpoints that legitimately return more (a full vulnerability catalogue, a
+ * maximum-size CVE page) pass their own larger cap explicitly.
+ */
+export const MAX_JSON_BODY_BYTES = 32 * 1024 * 1024;
+
+/**
+ * Cap for bulk catalogue downloads (CISA KEV's full feed, a 2000-item NVD page).
+ * These are legitimately tens of megabytes and grow over time, so they get their
+ * own ceiling rather than forcing the shared default up for everyone.
+ */
+export const MAX_BULK_JSON_BODY_BYTES = 96 * 1024 * 1024;
+
+/**
+ * Read a response body with a HARD byte cap, streaming chunk-by-chunk so an
+ * over-limit body aborts the moment the cap is crossed.
+ *
+ * A body-less response (test doubles, some mocks) falls back to `arrayBuffer()`;
+ * callers that had a post-read size check keep it as the belt for that path.
+ */
+export async function readBodyCapped(response: Response, maxBytes: number): Promise<Uint8Array> {
+  const body = response.body;
+  // Also tolerate non-standard doubles that expose a body without getReader.
+  if (!body || typeof body.getReader !== 'function') {
+    return new Uint8Array(await response.arrayBuffer());
+  }
+
+  const reader = body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value) {
+        total += value.byteLength;
+        if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
+        chunks.push(value);
+      }
+    }
+  } finally {
+    // Over-cap or downstream throw: release the connection instead of leaving
+    // the socket open (and the server still sending) until GC. No-op after a
+    // clean done=true read.
+    try { await reader.cancel(); } catch { /* already closed */ }
+  }
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const c of chunks) { out.set(c, offset); offset += c.byteLength; }
+  return out;
+}
+
+/**
+ * Whether this response can actually be read as a stream — i.e. whether the cap
+ * can be enforced at all.
+ *
+ * A real `Response` from `fetch()` always exposes a `ReadableStream` body except
+ * for the null-body statuses (204/205/304), which have nothing to cap. So in
+ * production this is true exactly when there are bytes to bound, and the capped
+ * path is never quietly skipped. It returns false only for the null-body statuses
+ * and for hand-rolled test doubles, which implement whichever of
+ * `arrayBuffer()`/`text()`/`json()` they need and no stream at all.
+ */
+function hasStreamBody(response: Response): boolean {
+  const body = response.body;
+  return !!body && typeof body.getReader === 'function';
+}
+
+/**
+ * Capped replacement for `response.text()`.
+ *
+ * Decodes UTF-8 non-fatally, matching `Response.text()` — a malformed byte becomes
+ * U+FFFD rather than throwing, so swapping this in cannot turn a body that used to
+ * parse into an exception.
+ */
+export async function readTextCapped(
+  response: Response,
+  maxBytes: number = MAX_JSON_BODY_BYTES,
+): Promise<string> {
+  if (hasStreamBody(response)) {
+    return new TextDecoder('utf-8').decode(await readBodyCapped(response, maxBytes));
+  }
+  // Double/null-body ladder — see hasStreamBody for why this is unreachable for a
+  // real body-bearing response.
+  if (typeof response.arrayBuffer === 'function') {
+    return new TextDecoder('utf-8').decode(new Uint8Array(await response.arrayBuffer()));
+  }
+  return response.text();
+}
+
+/**
+ * Capped replacement for `response.json()`.
+ *
+ * Throws {@link BodyTooLargeError} for an over-cap body and a `SyntaxError` for a
+ * malformed one, so callers that already distinguish "malformed JSON" from other
+ * failures keep working: the size failure is a distinct, catchable type rather
+ * than something that masquerades as a parse error.
+ */
+export async function readJsonCapped<T = unknown>(
+  response: Response,
+  maxBytes: number = MAX_JSON_BODY_BYTES,
+): Promise<T> {
+  if (hasStreamBody(response) || typeof response.arrayBuffer === 'function' || typeof response.text === 'function') {
+    return JSON.parse(await readTextCapped(response, maxBytes)) as T;
+  }
+  // A double that implements json() alone.
+  return response.json() as Promise<T>;
+}
+
+/**
+ * Accumulator for a Node `IncomingMessage` body that decodes UTF-8 exactly ONCE,
+ * at the end.
+ *
+ * The obvious `body += chunk` is silently lossy. `+=` stringifies each Buffer on
+ * its own, so a multi-byte character straddling a chunk boundary is decoded as two
+ * independent fragments and both halves become U+FFFD. Chunk boundaries fall
+ * wherever the socket happens to split, so the damage scales with payload size and
+ * lands only on non-ASCII text — measured at ~15 mangled characters in a 560KB CJK
+ * body. Nothing throws: `JSON.parse` still succeeds, so the corruption reaches the
+ * report and the knowledge store unannounced.
+ *
+ * Buffering the chunks and concatenating before a single decode is what makes a
+ * split sequence whole again.
+ *
+ * `byteLength` counts real bytes, which is also the honest unit for a payload cap —
+ * the old `body.length` measured UTF-16 code units of an already-corrupted string.
+ */
+export class Utf8Body {
+  private readonly chunks: Buffer[] = [];
+  private bytes = 0;
+
+  /** Total bytes accumulated so far. */
+  get byteLength(): number {
+    return this.bytes;
+  }
+
+  push(chunk: Buffer): void {
+    this.chunks.push(chunk);
+    this.bytes += chunk.byteLength;
+  }
+
+  /** Concatenate and decode. Safe to call once the stream has ended. */
+  toString(): string {
+    return Buffer.concat(this.chunks).toString('utf8');
+  }
+}

package/src/web-research/browser-search.ts

@@ -9,7 +9,7 @@ import { getMaxWorkers } from '../infrastructure/browser/config.ts';
 import { logger } from '../logger.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { normalizeUrl } from '../utils/url-utils.ts';
-import type { SearchResult } from './types.ts';
+import type { SearchResult, QueryFailure } from './types.ts';
 import type { Config } from '../config.ts';
 import { metrics } from '../utils/metrics.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
@@ -21,11 +21,20 @@ import type { ServiceContainer } from '../core/service-registry.ts';
  * Each worker maintains its own "warm" browser process.
  */
 export async function performSearch(
-    queries: string[], 
+    queries: string[],
     config?: Config,
     signal?: AbortSignal,
     onProgress?: (links: number) => void,
-    container: ServiceContainer = getServiceContainer()
+    container: ServiceContainer = getServiceContainer(),
+    /**
+     * Optional sink for per-query FAILURES (timeout / worker error). Populated as
+     * an out-param rather than folded into the return value because the returned
+     * map's `query -> results` shape is what every caller consumes; a query absent
+     * from this map produced zero results with no failure, i.e. the search really
+     * did come back empty. Without this the caller cannot tell those apart and
+     * reports every empty query as a too-narrow query.
+     */
+    failures?: Map<string, QueryFailure>,
 ): Promise<Map<string, SearchResult[]>> {
     const startTime = Date.now();
     const resultMap = new Map<string, SearchResult[]>();
@@ -78,7 +87,12 @@ export async function performSearch(
     // (post-dequeue) deadline.
     const QUERY_TIMEOUT_MS = (config?.SEARCH_TIMEOUT_MS ?? 45_000) + (config?.BROWSER_TASK_TIMEOUT_MS ?? 10_000);
 
-    const filteredQueries = queries.filter(q => q.trim());
+    // Deduplicated: resultMap and the failures map are both keyed by the query
+    // string, so two tasks for the same string race last-writer-wins — instance A
+    // delivering results, then instance B timing out, discarded A's results and
+    // marked the query failed. One task per distinct string closes that (and stops
+    // paying for the duplicate browser work).
+    const filteredQueries = [...new Set(queries.filter(q => q.trim()))];
     let timeoutCount = 0;
     let errorCount = 0;
     // Keep one representative worker error so total-failure surfaces the real
@@ -146,11 +160,31 @@ export async function performSearch(
             
             if (isTimeout) {
                 logger.warn(`[Search] Query timed out after ${QUERY_TIMEOUT_MS}ms — likely starved waiting for a free browser worker under concurrent load, or a slow/blocked search provider: "${query}"`);
+                failures?.set(query, {
+                    type: 'timeout',
+                    message: `Search timed out after ${QUERY_TIMEOUT_MS}ms before returning anything. This says nothing about the query — the browser pool was saturated or the search provider was slow/blocked. Retrying the same query later is reasonable.`,
+                });
             } else {
                 const msg = error instanceof Error ? error.message : String(error);
                 if (msg !== 'Aborted') {
                     if (!sampleWorkerError) sampleWorkerError = msg;
                     logger.error(`[Search] Worker failed for "${query}": ${msg}`);
+                    failures?.set(query, {
+                        type: 'service_unavailable',
+                        message: `The search backend failed for this query (${msg}). This is an infrastructure failure, not a signal about the query itself — do not rewrite the query in response to it.`,
+                    });
+                } else if (!signal?.aborted) {
+                    // A worker-side 'Aborted' with no caller abort (a pool drain or
+                    // leader handover cancelling the task) is an infrastructure
+                    // interruption. Without an entry here the query fell through to
+                    // the empty_results default — "the query may be too narrow" —
+                    // the exact misattribution this map exists to prevent. The
+                    // caller-abort case stays unrecorded: the post-loop signal
+                    // check reports the cancellation itself.
+                    failures?.set(query, {
+                        type: 'service_unavailable',
+                        message: 'The search task was cancelled on the worker side (pool drain or leader handover) before completing. This says nothing about the query — do not rewrite it; retrying the same query later is reasonable.',
+                    });
                 }
             }
             resultMap.set(query, []);
@@ -162,6 +196,18 @@ export async function performSearch(
 
     await Promise.all(searchTasks);
 
+    // A cancelled run is not a broken one. Every query aborts with zero results,
+    // which walks straight into the total-failure branch below and reports
+    // "Browser workers may be unavailable, DuckDuckGo is unreachable, or the
+    // system is under extreme load" — an infrastructure outage the user never had.
+    // They pressed Ctrl-C. Check this BEFORE the outage heuristic, and surface the
+    // codebase's standard abort signature so the callers that already special-case
+    // it (task-execution-service, the orchestrator, the CLI classifier) see a
+    // cancellation rather than a failure.
+    if (signal?.aborted) {
+        throw new Error('Aborted');
+    }
+
     // Detect total failure: if every valid query returned empty, the worker pool is likely dead.
     const totalResults = Array.from(resultMap.values()).reduce((sum, r) => sum + r.length, 0);
     if (totalResults === 0 && filteredQueries.length > 0) {

package/src/web-research/search.ts

@@ -6,7 +6,7 @@
 
 import { logger } from '../logger.ts';
 import { performSearch } from './browser-search.ts';
-import type { QueryResultWithError } from './types.ts';
+import type { QueryResultWithError, QueryFailure } from './types.ts';
 import type { Config } from '../config.ts';
 import { metrics } from '../utils/metrics.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
@@ -39,7 +39,10 @@ export async function search(
   const searchStart = Date.now();
   
   try {
-    const resultMap = await performSearch(queries, config, signal, onProgress, container);
+    // Per-query failures (timeout / dead worker) come back separately from the
+    // results so an empty result set can be attributed correctly below.
+    const failures = new Map<string, QueryFailure>();
+    const resultMap = await performSearch(queries, config, signal, onProgress, container, failures);
     const searchDuration = Date.now() - searchStart;
     metrics.observe('search_latency_ms', searchDuration);
     
@@ -56,7 +59,12 @@ export async function search(
         successfulQueries++;
         metrics.observe('search_results_per_query', qResults.length);
       } else {
-        result.error = {
+        // Only call it an empty result when the search actually completed and
+        // returned nothing. A recorded failure (timeout, dead worker, breaker
+        // fast-fail) carries its own message — telling the researcher "the query
+        // may be too narrow" for an infrastructure fault makes it rewrite a
+        // perfectly good query instead of retrying.
+        result.error = failures.get(q) ?? {
           type: 'empty_results',
           message: 'Browser-based search returned no relevant results for this query (the query may be too narrow, or the site returned nothing usable).'
         };
@@ -73,11 +81,18 @@ export async function search(
     return results;
   } catch (error) {
     const searchDuration = Date.now() - searchStart;
+    const message = error instanceof Error ? error.message : String(error);
+    // A cancelled run must propagate as a cancellation. Swallowing it into
+    // per-query `unknown` errors would let the orchestrator carry on and
+    // synthesize from nothing, then report a failure the user never had. Checked
+    // BEFORE the error metrics: a user cancelling runs is not a failing search
+    // backend, and counting each Esc as queries.length errors painted one.
+    if (signal?.aborted || message === 'Aborted') {
+      throw error;
+    }
     metrics.observe('search_latency_ms', searchDuration, { status: 'error' });
     metrics.increment('search_queries_total', queries.length, { status: 'error' });
     metrics.increment('search_errors_total', 1, { error_type: 'search_failed' });
-    
-    const message = error instanceof Error ? error.message : String(error);
     // Re-throw total search failure so the orchestrator can surface a clear error
     // rather than silently producing an empty synthesis with zero links.
     if (message.includes('Search completely failed')) {

package/src/web-research/types.ts

@@ -12,10 +12,23 @@ export type { SearchResult, ScrapeResult };
 export interface QueryResultWithError {
   query: string;
   results: SearchResult[];
-  error?: {
-    type: 'empty_results' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
-    message: string;
-  };
+  error?: QueryFailure;
+}
+
+/**
+ * Why one query yielded nothing.
+ *
+ * The distinction matters to the *researcher agent*, which reads these messages
+ * and decides what to do next: `empty_results` is genuine feedback about the
+ * query (rewrite it, broaden it), while a timeout or a dead worker says nothing
+ * at all about the query and should prompt a retry instead. Collapsing the two —
+ * reporting every zero-result query as "the query may be too narrow" — sent the
+ * agent off rewriting perfectly good queries whenever the browser pool was the
+ * thing that failed.
+ */
+export interface QueryFailure {
+  type: 'empty_results' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
+  message: string;
 }
 
 // Scraper types

package/src/web-research/web-scraper.ts

@@ -26,6 +26,7 @@ import {
 import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, formatErrorWithCause, isBenignScrapeFailure, } from './scraper-utils.ts';
 import { isTransientError, abortableDelay } from './retry-utils.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
+import { readBodyCapped, BodyTooLargeError } from '../utils/http-body.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
 import type { ServiceContainer } from '../core/service-registry.ts';
 import { cacheScrapedContent } from '../utils/shared-links.ts';
@@ -144,56 +145,12 @@ function decodeHtmlBody(bytes: Uint8Array, contentTypeHeader: string): string {
   }
 }
 
-/** Thrown by readBodyCapped when a streamed body crosses its byte cap. */
-class BodyTooLargeError extends Error {
-  constructor(public readonly bytesRead: number, public readonly maxBytes: number) {
-    super(`Response body exceeded ${maxBytes} byte cap`);
-    this.name = 'BodyTooLargeError';
-  }
-}
-
-/**
- * Read a response body with a HARD byte cap, streaming chunk-by-chunk so an
- * over-limit body aborts the moment the cap is crossed. The Content-Length
- * pre-screen above is advisory only: a chunked (or lying) response carries no
- * usable length header, and `response.arrayBuffer()` would otherwise buffer the
- * entire body — potentially hundreds of MB from a hostile server — into the
- * main process before any size check ran.
- *
- * A body-less response (test doubles, some mocks) falls back to arrayBuffer();
- * callers keep their post-read size checks as the belt for that path.
- */
-async function readBodyCapped(response: Response, maxBytes: number): Promise<Uint8Array> {
-  const body = response.body;
-  // Also tolerate non-standard doubles that expose a body without getReader.
-  if (!body || typeof body.getReader !== 'function') {
-    return new Uint8Array(await response.arrayBuffer());
-  }
-
-  const reader = body.getReader();
-  const chunks: Uint8Array[] = [];
-  let total = 0;
-  try {
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (value) {
-        total += value.byteLength;
-        if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
-        chunks.push(value);
-      }
-    }
-  } finally {
-    // Over-cap or downstream throw: release the connection instead of leaving
-    // the socket open (and the server still sending) until GC. No-op after a
-    // clean done=true read.
-    try { await reader.cancel(); } catch { /* already closed */ }
-  }
-  const out = new Uint8Array(total);
-  let offset = 0;
-  for (const c of chunks) { out.set(c, offset); offset += c.byteLength; }
-  return out;
-}
+// readBodyCapped / BodyTooLargeError live in utils/http-body.ts — the scraper
+// pioneered the streaming cap, and the JSON API clients now share that one
+// implementation instead of each carrying a near-copy. The Content-Length
+// pre-screen above stays advisory only: chunked responses carry no usable length
+// header and the header is freely falsifiable, so the streamed cap is the check
+// that actually bounds the allocation.
 
 /**
  * Record a fetch-layer failure exactly once per URL.

package/src/youtube/transcript-client.ts

@@ -21,6 +21,7 @@ import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-resea
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
 import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
+import { readTextCapped } from '../utils/http-body.ts';
 import { mintPoTokens } from './potoken.ts';
 import { watchUrl } from './video-id.ts';
 
@@ -255,7 +256,10 @@ async function fetchOne(
             void response.body?.cancel()?.catch(() => { /* best-effort */ });
             throw new Error(`HTTP ${response.status} from timedtext endpoint`);
           }
-          const body = await response.text();
+          // A timedtext document is kilobytes; the cap only bounds a hostile or
+          // malfunctioning response, which this path is directly exposed to because
+          // base_url comes out of a parsed getInfo() payload.
+          const body = await readTextCapped(response);
           const text = parseJson3(body);
 
           // Empty-body guard: HTTP 200 + empty body == silent PoToken failure.