@lincoln504/pi-research 1.1.2 → 1.3.0

package/src/utils/stdio-capture.ts

@@ -33,6 +33,26 @@ const sessionCaptureStates = new Map<string, boolean>();
 let unkeyedCaptureDepth = 0;
 let isAnyLoggerCapturingOutput = false;
 
+/**
+ * Active captures in creation order. Restoration MUST be LIFO: each capture saves
+ * the currently-installed writers/console/fs.writeSync (and the current FD-2
+ * target) as its "originals", so a frame that finishes while a later frame is
+ * still active must NOT restore — it would re-install the still-active frame's
+ * view of the world later, when that frame finally exits, re-pointing stderr,
+ * stdout, console.* and OS-level FD 2 at a dead capture's log-file plumbing for
+ * the remainder of the process. Keyed captures nest by design (a run's capture
+ * around the embedding server's), and nothing orders their exits — a cancelled
+ * run's outer capture routinely ends while the embedder-init capture is mid
+ * flight. So: an out-of-order finisher only marks its frame retired; the frame
+ * on top of the stack performs the actual restores, unwinding every already-
+ * retired frame beneath it in strict reverse order.
+ */
+interface CaptureFrame {
+  retired: boolean;
+  restore: () => void;
+}
+const captureStack: CaptureFrame[] = [];
+
 /**
  * Recompute the global "is anything capturing" flag from BOTH the keyed sessions and
  * the unkeyed-capture depth, so neither scheme can clobber the other's state.
@@ -220,7 +240,9 @@ export async function captureStdio<T>(
       if (shouldRedirectStderr(stat)) {
         savedFd2 = fs.openSync('/dev/fd/2', 'a');
         fs.closeSync(2);
-        const newFd = fs.openSync(logFile, 'a');
+        // 0o600 applies only when this open CREATES the file — keeps the log
+        // owner-only in a world-traversable tmpdir if rotation removed it.
+        const newFd = fs.openSync(logFile, 'a', 0o600);
         if (newFd === 2) {
           fd2Redirected = true;
         } else {
@@ -371,20 +393,41 @@ export async function captureStdio<T>(
     }
   } catch (_e) { /* ignore */ }
 
+  const frame: CaptureFrame = {
+    retired: false,
+    restore: () => {
+      process.stderr.write = originalStderrWrite;
+      process.stdout.write = originalStdoutWrite;
+      try {
+        fs.writeSync = originalFsWriteSync;
+      } catch { /* ignore */ }
+      console.log = originalConsole.log;
+      console.info = originalConsole.info;
+      console.warn = originalConsole.warn;
+      console.error = originalConsole.error;
+      console.debug = originalConsole.debug;
+
+      // Restore FD 2 to what it was when THIS frame started (for a nested frame
+      // that is the enclosing frame's log-file redirect; the enclosing frame's
+      // own restore, which always runs after this one in the unwind, then puts
+      // the true terminal FD back).
+      if (fd2Redirected && savedFd2 >= 0) {
+        try {
+          fs.closeSync(2);
+          const r = fs.openSync(`/dev/fd/${savedFd2}`, 'a');
+          if (r !== 2 && r >= 0) {
+            // Didn't recover FD 2 — leave restored FD open
+          }
+          try { fs.closeSync(savedFd2); } catch { /* ignore */ }
+        } catch { /* ignore restore errors */ }
+      }
+    },
+  };
+  captureStack.push(frame);
+
   try {
     return await task();
   } finally {
-    // Restore everything
-    process.stderr.write = originalStderrWrite;
-    process.stdout.write = originalStdoutWrite;
-    try {
-      fs.writeSync = originalFsWriteSync;
-    } catch { /* ignore */ }
-    console.log = originalConsole.log;
-    console.info = originalConsole.info;
-    console.warn = originalConsole.warn;
-    console.error = originalConsole.error;
-    console.debug = originalConsole.debug;
     if (sessionId) {
       setSessionCapturing(sessionId, false);
     } else {
@@ -392,16 +435,14 @@ export async function captureStdio<T>(
       recomputeGlobalCapturing();
     }
 
-    // Restore FD 2
-    if (fd2Redirected && savedFd2 >= 0) {
-      try {
-        fs.closeSync(2);
-        const r = fs.openSync(`/dev/fd/${savedFd2}`, 'a');
-        if (r !== 2 && r >= 0) {
-          // Didn't recover FD 2 — leave restored FD open
-        }
-        try { fs.closeSync(savedFd2); } catch { /* ignore */ }
-      } catch { /* ignore restore errors */ }
+    // LIFO unwind (see captureStack): only the top frame restores; an
+    // out-of-order finisher parks as retired and is unwound — in reverse
+    // order — when everything above it has finished. Until then its patches
+    // stay installed, which merely extends capture a little; restoring early
+    // would divert stdio permanently.
+    frame.retired = true;
+    while (captureStack.length > 0 && captureStack[captureStack.length - 1]!.retired) {
+      captureStack.pop()!.restore();
     }
   }
 }

package/src/utils/text-utils.ts

@@ -76,7 +76,14 @@ export function truncateWithMarker(content: string, maxChars: number): string {
   const makeMarker = (kept: number) => `\n\n[content truncated: showing ${kept} of ${total} chars]`;
   // Reserve marker room using the longest possible marker (kept ≤ total, so the
   // digit count of `kept` never exceeds that of `total`).
-  const keep = Math.max(0, maxChars - makeMarker(total).length);
+  let keep = Math.max(0, maxChars - makeMarker(total).length);
+  // slice() counts UTF-16 code units, so the cut can land between the halves of a
+  // surrogate pair — a lone high surrogate in the body is invalid text that some
+  // providers reject (400) when it reaches a JSON payload. Backing off one unit
+  // restores well-formed text. The whitespace-boundary path below cannot split a
+  // pair: it cuts immediately before a BMP whitespace character.
+  const lastKeptUnit = content.charCodeAt(keep - 1);
+  if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
   let body = content.slice(0, keep);
   // Prefer a clean boundary near the cut point.
   const boundary = Math.max(body.lastIndexOf('\n'), body.lastIndexOf(' '));

package/src/utils/tool-usage-tracker.ts

@@ -125,13 +125,20 @@ export class ToolUsageTracker {
     
     const toolLimit = this.limits[toolName as keyof ToolLimits];
     const catLimit = usage.limit;
+    // Decide by which limit actually tripped, computed exactly as recordCall
+    // computes it: a first-ever `search` blocked by the shared gathering cap must
+    // NOT claim "you have already used your one search call" — the agent would
+    // wrongly believe it had searched. The per-tool wording is only correct when
+    // the per-tool counter itself is at its limit.
+    const toolCount = usage.toolCounts.get(toolName) || 0;
+    const toolLimitTripped = toolLimit !== undefined && toolCount >= toolLimit;
 
-    if (toolName === 'search' && toolLimit === 1) {
+    if (toolName === 'search' && toolLimit === 1 && toolLimitTripped) {
         return `SEARCH LIMIT REACHED: You have already used your one search call. ` +
             `Proceed to scraping: use the scrape tool for full deep-dives into your best search results.`;
     }
 
-    if (toolName === 'youtube_transcript' && toolLimit === 1) {
+    if (toolName === 'youtube_transcript' && toolLimit === 1 && toolLimitTripped) {
         return `YOUTUBE TRANSCRIPT LIMIT REACHED: You have already used your one youtube_transcript call. ` +
             `Continue with your other gathering tools (search, scrape) and proceed to synthesis.`;
     }

package/src/web-research/browser-search.ts

@@ -9,7 +9,7 @@ import { getMaxWorkers } from '../infrastructure/browser/config.ts';
 import { logger } from '../logger.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { normalizeUrl } from '../utils/url-utils.ts';
-import type { SearchResult } from './types.ts';
+import type { SearchResult, QueryFailure } from './types.ts';
 import type { Config } from '../config.ts';
 import { metrics } from '../utils/metrics.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
@@ -21,11 +21,28 @@ import type { ServiceContainer } from '../core/service-registry.ts';
  * Each worker maintains its own "warm" browser process.
  */
 export async function performSearch(
-    queries: string[], 
+    queries: string[],
     config?: Config,
     signal?: AbortSignal,
     onProgress?: (links: number) => void,
-    container: ServiceContainer = getServiceContainer()
+    container: ServiceContainer = getServiceContainer(),
+    /**
+     * Optional sink for per-query FAILURES (timeout / worker error / cross-query
+     * dedup). Populated as an out-param rather than folded into the return value
+     * because the returned map's `query -> results` shape is what every caller
+     * consumes; a query absent from this map produced zero results with no
+     * failure, i.e. the search really did come back empty. Without this the
+     * caller cannot tell those apart and reports every empty query as a
+     * too-narrow query.
+     */
+    failures?: Map<string, QueryFailure>,
+    /**
+     * Session/research id keying the per-session circuit breaker inside
+     * runWorkerSearch — mirroring the scrape path, which threads it via
+     * BrowserTask.sessionId. Callers without a session (undefined) share the
+     * global breaker, as before.
+     */
+    sessionId?: string,
 ): Promise<Map<string, SearchResult[]>> {
     const startTime = Date.now();
     const resultMap = new Map<string, SearchResult[]>();
@@ -78,7 +95,12 @@ export async function performSearch(
     // (post-dequeue) deadline.
     const QUERY_TIMEOUT_MS = (config?.SEARCH_TIMEOUT_MS ?? 45_000) + (config?.BROWSER_TASK_TIMEOUT_MS ?? 10_000);
 
-    const filteredQueries = queries.filter(q => q.trim());
+    // Deduplicated: resultMap and the failures map are both keyed by the query
+    // string, so two tasks for the same string race last-writer-wins — instance A
+    // delivering results, then instance B timing out, discarded A's results and
+    // marked the query failed. One task per distinct string closes that (and stops
+    // paying for the duplicate browser work).
+    const filteredQueries = [...new Set(queries.filter(q => q.trim()))];
     let timeoutCount = 0;
     let errorCount = 0;
     // Keep one representative worker error so total-failure surfaces the real
@@ -102,7 +124,7 @@ export async function performSearch(
               ? AbortSignal.any([signal, timeoutController.signal])
               : timeoutController.signal;
 
-            const results = await runWorkerSearch(query, config, querySignal, 1, undefined, container);
+            const results = await runWorkerSearch(query, config, querySignal, 1, sessionId, container);
             const queryDuration = Date.now() - queryStartTime;
             metrics.observe('browser_search_query_duration_ms', queryDuration);
 
@@ -128,6 +150,17 @@ export async function performSearch(
                         uniqueResults.push(r);
                     }
                 }
+                if (uniqueResults.length === 0) {
+                    // Every raw result deduplicated away against URLs earlier
+                    // queries in this batch already returned. Without an entry
+                    // here the query fell through to the empty_results default —
+                    // "the query may be too narrow" — sending the researcher off
+                    // rewriting a query that actually worked.
+                    failures?.set(query, {
+                        type: 'all_duplicates',
+                        message: `All ${results.length} results for this query duplicated URLs already returned by earlier queries in this batch. The query succeeded but surfaced nothing new — do not rewrite it; try a different angle only if you still need additional sources.`,
+                    });
+                }
                 resultMap.set(query, uniqueResults);
             } else {
                 metrics.increment('browser_search_queries_total', 1, { status: 'no_results' });
@@ -146,11 +179,31 @@ export async function performSearch(
             
             if (isTimeout) {
                 logger.warn(`[Search] Query timed out after ${QUERY_TIMEOUT_MS}ms — likely starved waiting for a free browser worker under concurrent load, or a slow/blocked search provider: "${query}"`);
+                failures?.set(query, {
+                    type: 'timeout',
+                    message: `Search timed out after ${QUERY_TIMEOUT_MS}ms before returning anything. This says nothing about the query — the browser pool was saturated or the search provider was slow/blocked. Retrying the same query later is reasonable.`,
+                });
             } else {
                 const msg = error instanceof Error ? error.message : String(error);
                 if (msg !== 'Aborted') {
                     if (!sampleWorkerError) sampleWorkerError = msg;
                     logger.error(`[Search] Worker failed for "${query}": ${msg}`);
+                    failures?.set(query, {
+                        type: 'service_unavailable',
+                        message: `The search backend failed for this query (${msg}). This is an infrastructure failure, not a signal about the query itself — do not rewrite the query in response to it.`,
+                    });
+                } else if (!signal?.aborted) {
+                    // A worker-side 'Aborted' with no caller abort (a pool drain or
+                    // leader handover cancelling the task) is an infrastructure
+                    // interruption. Without an entry here the query fell through to
+                    // the empty_results default — "the query may be too narrow" —
+                    // the exact misattribution this map exists to prevent. The
+                    // caller-abort case stays unrecorded: the post-loop signal
+                    // check reports the cancellation itself.
+                    failures?.set(query, {
+                        type: 'service_unavailable',
+                        message: 'The search task was cancelled on the worker side (pool drain or leader handover) before completing. This says nothing about the query — do not rewrite it; retrying the same query later is reasonable.',
+                    });
                 }
             }
             resultMap.set(query, []);
@@ -162,6 +215,18 @@ export async function performSearch(
 
     await Promise.all(searchTasks);
 
+    // A cancelled run is not a broken one. Every query aborts with zero results,
+    // which walks straight into the total-failure branch below and reports
+    // "Browser workers may be unavailable, DuckDuckGo is unreachable, or the
+    // system is under extreme load" — an infrastructure outage the user never had.
+    // They pressed Ctrl-C. Check this BEFORE the outage heuristic, and surface the
+    // codebase's standard abort signature so the callers that already special-case
+    // it (task-execution-service, the orchestrator, the CLI classifier) see a
+    // cancellation rather than a failure.
+    if (signal?.aborted) {
+        throw new Error('Aborted');
+    }
+
     // Detect total failure: if every valid query returned empty, the worker pool is likely dead.
     const totalResults = Array.from(resultMap.values()).reduce((sum, r) => sum + r.length, 0);
     if (totalResults === 0 && filteredQueries.length > 0) {

package/src/web-research/scraper-utils.ts

@@ -343,6 +343,11 @@ function isPrivateIpv6(ip: string): boolean {
   // Link-local (fe80::/10)
   if (normalized.startsWith('fe8') || normalized.startsWith('fe9') ||
       normalized.startsWith('fea') || normalized.startsWith('feb')) return true;
+  // Deprecated site-local (fec0::/10, RFC 3879, deprecated 2004). No OS
+  // auto-routes to it today, but it's still squarely "private/reserved"
+  // address space — the sibling range immediately above this one.
+  if (normalized.startsWith('fec') || normalized.startsWith('fed') ||
+      normalized.startsWith('fee') || normalized.startsWith('fef')) return true;
   // IPv4-mapped ::ffff:x.x.x.x — check the embedded IPv4 part. The mapped tail
   // can arrive dotted-decimal (::ffff:127.0.0.1) OR hex (::ffff:7f00:1) — Node's
   // URL parser normalizes literals to the hex form — and the prefix can be
@@ -357,6 +362,14 @@ function isPrivateIpv6(ip: string): boolean {
   // legitimate scrape target, so block the whole ranges rather than decode them.
   if (normalized.startsWith('2002:')) return true;
   if (/^2001:0{0,4}:/.test(normalized) || normalized.startsWith('2001::')) return true;
+  // NAT64 well-known prefix (64:ff9b::/96, RFC 6052) also tunnels an IPv4
+  // address, and unlike the transitional ranges above it's actively deployed
+  // today (464XLAT on IPv6-only mobile/enterprise networks). A host whose
+  // outbound path traverses a NAT64 gateway gets this transparently
+  // translated to a direct connection to the embedded IPv4 (e.g. the cloud
+  // metadata service) — block the whole prefix regardless of what it wraps,
+  // same rationale as the 6to4/Teredo ranges just above.
+  if (normalized.startsWith('64:ff9b:')) return true;
   // All-zeros (unspecified address)
   if (normalized === '::' || normalized === '0:0:0:0:0:0:0:0') return true;
   return false;

package/src/web-research/search.ts

@@ -6,7 +6,7 @@
 
 import { logger } from '../logger.ts';
 import { performSearch } from './browser-search.ts';
-import type { QueryResultWithError } from './types.ts';
+import type { QueryResultWithError, QueryFailure } from './types.ts';
 import type { Config } from '../config.ts';
 import { metrics } from '../utils/metrics.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
@@ -23,6 +23,9 @@ import type { ServiceContainer } from '../core/service-registry.ts';
  * @param config - Optional configuration override
  * @param signal - Optional AbortSignal
  * @param onProgress - Optional callback with cumulative link count across completed queries
+ * @param container - Service container for scheduler resolution
+ * @param sessionId - Research/session id keying the per-session circuit breaker
+ *   (mirrors the scrape path); undefined callers share the global breaker
  * @returns Promise<QueryResultWithError[]> - Array of search results with error information
  */
 export async function search(
@@ -30,7 +33,8 @@ export async function search(
   config?: Config,
   signal?: AbortSignal,
   onProgress?: (links: number) => void,
-  container: ServiceContainer = getServiceContainer()
+  container: ServiceContainer = getServiceContainer(),
+  sessionId?: string
 ): Promise<QueryResultWithError[]> {
   if (queries.length === 0) return [];
   
@@ -39,7 +43,10 @@ export async function search(
   const searchStart = Date.now();
   
   try {
-    const resultMap = await performSearch(queries, config, signal, onProgress, container);
+    // Per-query failures (timeout / dead worker) come back separately from the
+    // results so an empty result set can be attributed correctly below.
+    const failures = new Map<string, QueryFailure>();
+    const resultMap = await performSearch(queries, config, signal, onProgress, container, failures, sessionId);
     const searchDuration = Date.now() - searchStart;
     metrics.observe('search_latency_ms', searchDuration);
     
@@ -56,7 +63,12 @@ export async function search(
         successfulQueries++;
         metrics.observe('search_results_per_query', qResults.length);
       } else {
-        result.error = {
+        // Only call it an empty result when the search actually completed and
+        // returned nothing. A recorded failure (timeout, dead worker, breaker
+        // fast-fail) carries its own message — telling the researcher "the query
+        // may be too narrow" for an infrastructure fault makes it rewrite a
+        // perfectly good query instead of retrying.
+        result.error = failures.get(q) ?? {
           type: 'empty_results',
           message: 'Browser-based search returned no relevant results for this query (the query may be too narrow, or the site returned nothing usable).'
         };
@@ -73,11 +85,18 @@ export async function search(
     return results;
   } catch (error) {
     const searchDuration = Date.now() - searchStart;
+    const message = error instanceof Error ? error.message : String(error);
+    // A cancelled run must propagate as a cancellation. Swallowing it into
+    // per-query `unknown` errors would let the orchestrator carry on and
+    // synthesize from nothing, then report a failure the user never had. Checked
+    // BEFORE the error metrics: a user cancelling runs is not a failing search
+    // backend, and counting each Esc as queries.length errors painted one.
+    if (signal?.aborted || message === 'Aborted') {
+      throw error;
+    }
     metrics.observe('search_latency_ms', searchDuration, { status: 'error' });
     metrics.increment('search_queries_total', queries.length, { status: 'error' });
     metrics.increment('search_errors_total', 1, { error_type: 'search_failed' });
-    
-    const message = error instanceof Error ? error.message : String(error);
     // Re-throw total search failure so the orchestrator can surface a clear error
     // rather than silently producing an empty synthesis with zero links.
     if (message.includes('Search completely failed')) {

package/src/web-research/types.ts

@@ -12,10 +12,25 @@ export type { SearchResult, ScrapeResult };
 export interface QueryResultWithError {
   query: string;
   results: SearchResult[];
-  error?: {
-    type: 'empty_results' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
-    message: string;
-  };
+  error?: QueryFailure;
+}
+
+/**
+ * Why one query yielded nothing.
+ *
+ * The distinction matters to the *researcher agent*, which reads these messages
+ * and decides what to do next: `empty_results` is genuine feedback about the
+ * query (rewrite it, broaden it), while a timeout or a dead worker says nothing
+ * at all about the query and should prompt a retry instead. Collapsing the two —
+ * reporting every zero-result query as "the query may be too narrow" — sent the
+ * agent off rewriting perfectly good queries whenever the browser pool was the
+ * thing that failed. `all_duplicates` is the third case: the query SUCCEEDED,
+ * but every result deduplicated away against URLs earlier queries in the batch
+ * already returned — coverage feedback, not query-quality feedback.
+ */
+export interface QueryFailure {
+  type: 'empty_results' | 'all_duplicates' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
+  message: string;
 }
 
 // Scraper types

package/src/web-research/web-scraper.ts

@@ -26,6 +26,7 @@ import {
 import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, formatErrorWithCause, isBenignScrapeFailure, } from './scraper-utils.ts';
 import { isTransientError, abortableDelay } from './retry-utils.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
+import { readBodyCapped, BodyTooLargeError } from '../utils/http-body.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
 import type { ServiceContainer } from '../core/service-registry.ts';
 import { cacheScrapedContent } from '../utils/shared-links.ts';
@@ -79,7 +80,11 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
     const sizeMB = Math.round(bytes.length / 1024 / 1024);
     logger.warn(`[Scrapers] PDF too large (${sizeMB}MB, max 100MB), skipping extraction`);
     metrics.increment('scrape_pdf_errors_total', 1, { error_type: 'size_exceeded' });
-    return `*Error: PDF too large (${sizeMB}MB, max 100MB).*`;
+    // THROW, never an in-band "*Error: ...*" string: a returned banner flowed
+    // into validateContent as if it were page content, and a verbose one
+    // (≥50 words / ≥200 chars) cleared the stub gate — the failure banner was
+    // then cached and cited as a successful scrape.
+    throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
   }
 
   const pdfExtractionStart = Date.now();
@@ -113,7 +118,9 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
       contentType: 'pdf',
       errorType: 'extraction_failed',
     });
-    return `*Error: Could not extract content from PDF (${msg}).*`;
+    // See the size branch above: an extraction failure must FAIL the scrape,
+    // not travel in-band as pseudo-content.
+    throw new Error(`Could not extract content from PDF (${msg})`, { cause: e });
   }
 }
 
@@ -144,56 +151,12 @@ function decodeHtmlBody(bytes: Uint8Array, contentTypeHeader: string): string {
   }
 }
 
-/** Thrown by readBodyCapped when a streamed body crosses its byte cap. */
-class BodyTooLargeError extends Error {
-  constructor(public readonly bytesRead: number, public readonly maxBytes: number) {
-    super(`Response body exceeded ${maxBytes} byte cap`);
-    this.name = 'BodyTooLargeError';
-  }
-}
-
-/**
- * Read a response body with a HARD byte cap, streaming chunk-by-chunk so an
- * over-limit body aborts the moment the cap is crossed. The Content-Length
- * pre-screen above is advisory only: a chunked (or lying) response carries no
- * usable length header, and `response.arrayBuffer()` would otherwise buffer the
- * entire body — potentially hundreds of MB from a hostile server — into the
- * main process before any size check ran.
- *
- * A body-less response (test doubles, some mocks) falls back to arrayBuffer();
- * callers keep their post-read size checks as the belt for that path.
- */
-async function readBodyCapped(response: Response, maxBytes: number): Promise<Uint8Array> {
-  const body = response.body;
-  // Also tolerate non-standard doubles that expose a body without getReader.
-  if (!body || typeof body.getReader !== 'function') {
-    return new Uint8Array(await response.arrayBuffer());
-  }
-
-  const reader = body.getReader();
-  const chunks: Uint8Array[] = [];
-  let total = 0;
-  try {
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (value) {
-        total += value.byteLength;
-        if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
-        chunks.push(value);
-      }
-    }
-  } finally {
-    // Over-cap or downstream throw: release the connection instead of leaving
-    // the socket open (and the server still sending) until GC. No-op after a
-    // clean done=true read.
-    try { await reader.cancel(); } catch { /* already closed */ }
-  }
-  const out = new Uint8Array(total);
-  let offset = 0;
-  for (const c of chunks) { out.set(c, offset); offset += c.byteLength; }
-  return out;
-}
+// readBodyCapped / BodyTooLargeError live in utils/http-body.ts — the scraper
+// pioneered the streaming cap, and the JSON API clients now share that one
+// implementation instead of each carrying a near-copy. The Content-Length
+// pre-screen above stays advisory only: chunked responses carry no usable length
+// header and the header is freely falsifiable, so the streamed cap is the check
+// that actually bounds the allocation.
 
 /**
  * Record a fetch-layer failure exactly once per URL.
@@ -397,8 +360,30 @@ async function scrapeWithStealthBrowser(_url: string, config?: Config, signal?:
     const result = await runBrowserTask<any>({ url: _url, sessionId }, 'scrape', config, signal, 1, container);
     const browserDuration = Date.now() - browserStart;
 
-    if (result.buffer) {
-      const markdown = await extractPdfToMarkdown(new Uint8Array(result.buffer));
+    // PDF bytes travel base64-encoded (bufferB64): the worker result crosses the
+    // cluster-IPC JSON hop (and, for followers, a second JSON hop through the
+    // leader's browser-server), which turns a raw Buffer into
+    // {type:'Buffer',data:[...]} — new Uint8Array(that) is ZERO bytes, so the
+    // extraction ran on nothing and its error string was cached as a "success".
+    // The JSON-roundtripped legacy shape is still decoded so a mixed-version
+    // leader/follower pair keeps working.
+    const legacyBuffer = result.buffer;
+    const pdfBytes: Buffer | null =
+      typeof result.bufferB64 === 'string'
+        ? Buffer.from(result.bufferB64, 'base64')
+        : Array.isArray(legacyBuffer?.data)
+          ? Buffer.from(legacyBuffer.data)
+          : legacyBuffer instanceof Uint8Array
+            ? Buffer.from(legacyBuffer)
+            : null;
+
+    if (pdfBytes !== null) {
+      const markdown = await extractPdfToMarkdown(pdfBytes);
+      // Mirror the fetch layer's PDF branch: extraction failures THROW from
+      // extractPdfToMarkdown, and validateContent still fails a nominally
+      // successful extraction that produced no real content — neither may be
+      // recorded (and cached) as a success.
+      validateContent('', markdown, _url);
       metrics.increment('scrape_operations_total', 1, { layer: 'playwright', content_type: 'pdf', status: 'success' });
       metrics.observe('scrape_latency_ms', browserDuration, { layer: 'playwright', content_type: 'pdf', status: 'success' });
       return { source: 'playwright', layer: 'playwright+camoufox', markdown };
@@ -429,6 +414,10 @@ async function scrapeWithStealthBrowser(_url: string, config?: Config, signal?:
     metrics.observe('scrape_latency_ms', browserDuration, { layer: 'playwright', content_type: 'html', status: 'success' });
     return { source: 'playwright', layer: 'playwright+camoufox', markdown };
   } catch (error) {
+    // A caller cancellation is not a scrape failure: no layer error metrics and
+    // no tracker entry — the search path applies the same 'Aborted' special case
+    // end-to-end (see browser-search.ts). scrapeSingle classifies it above us.
+    if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw error;
     const browserDuration = Date.now() - browserStart;
     metrics.increment('scrape_operations_total', 1, { layer: 'playwright', status: 'error' });
     metrics.observe('scrape_latency_ms', browserDuration, { layer: 'playwright', status: 'error' });
@@ -444,6 +433,15 @@ async function scrapeWithStealthBrowser(_url: string, config?: Config, signal?:
 }
 
 export async function scrapeSingle(url: string, signal?: AbortSignal, config?: Config, sessionId?: string, container: ServiceContainer = getServiceContainer()): Promise<ScrapeResult> {
+  // Cancellation short-circuit: after a user abort each remaining URL would
+  // otherwise still pay for SSRF DNS validation, a doomed fetch, and a browser
+  // fallback — all logged/counted as failures the user never had. 'Aborted' is
+  // the codebase's recognized cancellation signature (see browser-search.ts);
+  // deliberately no total_failure tally, no tracker entry, no ERROR log.
+  if (signal?.aborted) {
+    return { url, success: false, error: 'Aborted', markdown: '' };
+  }
+
   // Guard against an array accidentally stringified and passed as a single url (JSON arrays start
   // with '['). Do NOT reject brackets ANYWHERE: valid URLs legitimately contain them in query
   // strings (e.g. ?ids[]=1) and IPv6 literals (http://[::1]/) — the old includes('[') check dropped
@@ -520,6 +518,13 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
 
     return { ...res, url, success: true };
   } catch (e1) {
+    // Cancelled mid-fetch: not a fetch-layer failure, and no reason to pay for
+    // the browser fallback — classify as the recognized cancellation (no ERROR
+    // log, no tracker entry, no total_failure tally).
+    if (signal?.aborted || (e1 instanceof Error && e1.message === 'Aborted')) {
+      logger.debug(`[Scrapers] Scrape cancelled for ${url}`);
+      return { url, success: false, error: 'Aborted', markdown: '' };
+    }
     const fetchDuration = Date.now() - start;
     logger.debug(`[Scrapers] fetch failed for ${url} in ${fetchDuration}ms: ${formatErrorWithCause(e1)}`);
     // Tracked here rather than inside scrapeWithFetch, which is retried above —
@@ -542,6 +547,14 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
 
         return { ...res, url, success: true };
       } catch (e2) {
+        // See the fetch-layer catch above: a cancellation surfacing from the
+        // browser layer (runBrowserTask throws exactly 'Aborted' on a caller
+        // abort) is not a failure — it must not reach the ERROR log, the error
+        // tracker, or the total_failure tally.
+        if (signal?.aborted || (e2 instanceof Error && e2.message === 'Aborted')) {
+          logger.debug(`[Scrapers] Browser scrape cancelled for ${url}`);
+          return { url, success: false, error: 'Aborted', markdown: '' };
+        }
         const totalDuration = Date.now() - start;
         // A blocked/stubbed/HTTP-errored/slow/pool-draining URL is an expected
         // scraping outcome, already surfaced to the user as "not scraped" (the
@@ -573,6 +586,11 @@ export async function scrape(urls: string[], maxConcurrency = 5, signal?: AbortS
   
   const results: ScrapeResult[] = [];
   for (let i = 0; i < urls.length; i += maxConcurrency) {
+    // Stop dispatching once the caller has cancelled; URLs already in flight
+    // settle via scrapeSingle's own cancellation classification. Without this,
+    // every remaining batch still ran its full fetch + browser-fallback cycle
+    // after a user cancel.
+    if (signal?.aborted) break;
     const batch = urls.slice(i, i + maxConcurrency);
     const batchRes = await Promise.all(
       batch.map(async url => {

package/src/youtube/transcript-client.ts

@@ -21,6 +21,7 @@ import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-resea
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
 import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
+import { readTextCapped } from '../utils/http-body.ts';
 import { mintPoTokens } from './potoken.ts';
 import { watchUrl } from './video-id.ts';
 
@@ -255,7 +256,10 @@ async function fetchOne(
             void response.body?.cancel()?.catch(() => { /* best-effort */ });
             throw new Error(`HTTP ${response.status} from timedtext endpoint`);
           }
-          const body = await response.text();
+          // A timedtext document is kilobytes; the cap only bounds a hostile or
+          // malfunctioning response, which this path is directly exposed to because
+          // base_url comes out of a parsed getInfo() payload.
+          const body = await readTextCapped(response);
           const text = parseJson3(body);
 
           // Empty-body guard: HTTP 200 + empty body == silent PoToken failure.