@lincoln504/pi-research 1.2.0 → 1.3.0

package/src/utils/log-rotation.ts

@@ -27,8 +27,19 @@ export class LogRotation {
     try {
       if (fs.existsSync(logFile)) {
         fs.unlinkSync(logFile);
+        // Same rationale as the post-rotation recreate in rotateLogFile: the
+        // unlink discarded the 0o600 file, and a umask-default recreate by the
+        // next append would drop the owner-only posture.
+        try {
+          fs.closeSync(fs.openSync(logFile, 'a', 0o600));
+        } catch { /* best-effort — the next append recreates the file */ }
+        // And, as there, re-stamp the mode: a concurrent append landing between
+        // the unlink and the open makes the creation-time 0o600 a no-op.
+        try {
+          fs.chmodSync(logFile, 0o600);
+        } catch { /* best-effort */ }
       }
-      
+
       const files = fs.readdirSync(logDir);
       const baseName = path.basename(logFile);
       const archives = files.filter(f => f.startsWith(baseName) && f !== baseName);
@@ -63,19 +74,40 @@ export class LogRotation {
       const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
       const archivePath = `${logFile}.${timestamp}`;
       
-      // Rename current log file to archive
+      // Rename current log file to archive. The archive name is timestamped, so
+      // the target never pre-exists — a rename failure means the SOURCE is held
+      // open (Windows sharing semantics). A copy+unlink fallback here would
+      // destroy appends the OTHER processes writing this consolidated log (main,
+      // embedding server, browser workers) land between the copy and the unlink.
+      // Skipping the rotation is the data-preserving choice: the file transiently
+      // exceeds the cap until a later check succeeds (accepted by the throttling
+      // contract in checkAndRotate).
       try {
         fs.renameSync(logFile, archivePath);
       } catch (renameErr) {
-        // fs.renameSync fails on Windows if target exists (NTFS). Fall back to copy+delete.
-        if (process.platform === 'win32') {
-          fs.copyFileSync(logFile, archivePath);
-          fs.unlinkSync(logFile);
-        } else {
-          throw renameErr;
-        }
+        this.logger.log('[Logger] Log rotation skipped (rename failed; file likely in use):', renameErr);
+        return;
       }
-      
+
+      // Rotation moved the 0o600 file aside; without an explicit recreate the next
+      // append materializes it at the umask default (typically 0644) in a
+      // world-traversable tmpdir, silently dropping the owner-only posture Logger
+      // establishes at construction. Mode applies only at creation, so this is
+      // harmless where a file already exists or mode semantics differ (win32).
+      try {
+        fs.closeSync(fs.openSync(logFile, 'a', 0o600));
+      } catch { /* best-effort — the next append recreates the file */ }
+      // The mode above applies only if THIS open created the file. Several
+      // processes append to this consolidated log, so a concurrent async append
+      // can recreate it at umask perms between the rename and the open, turning
+      // the 0o600 into a no-op — stamp the mode unconditionally to close that
+      // window for good. ENOENT (nothing recreated it and the open failed) is
+      // tolerable: best-effort posture, same as the recreate itself.
+      try {
+        fs.chmodSync(logFile, 0o600);
+      } catch { /* best-effort */ }
+
+
       // Clean up old archives
       try {
         const files = fs.readdirSync(logDir);

package/src/utils/log-utils.ts

@@ -86,6 +86,11 @@ const MAX_LOG_MESSAGE_LENGTH = 10_000;
 const ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
 // Credentials embedded in a URL userinfo component: scheme://user:pass@host
 const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
+// Cookie / Set-Cookie headers. The KV value class below stops at whitespace, so
+// "Cookie: lang=en; SID=secret123" would mask only the first pair and leave every
+// later one exposed. A cookie string is a ;-and-space separated list of secrets,
+// not a single token — mask the ENTIRE header value to end-of-line.
+const COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
 // key=value / key: "value" pairs whose key names a secret.
 const SENSITIVE_KV_PATTERN =
   /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
@@ -137,6 +142,9 @@ export function redactSecrets(message: string): string {
   // Before the KV pass, for the same reason as Basic above: the KV value class
   // stops at the space after "Bearer", leaving an opaque token exposed.
   out = out.replace(BEARER_AUTH_PATTERN, 'Bearer [REDACTED]');
+  // Whole-header cookie masking before the KV pass, which would otherwise stop
+  // at the first whitespace and leave every subsequent pair in the clear.
+  out = out.replace(COOKIE_HEADER_PATTERN, (_m, key: string, sep: string) => `${key}${sep}[REDACTED]`);
   out = out.replace(SENSITIVE_KV_PATTERN, (_m, key: string, sep: string) => `${key}${sep}[REDACTED]`);
   out = out.replace(KNOWN_TOKEN_PATTERN, '[REDACTED]');
   out = out.replace(PROVIDER_TOKEN_PATTERN, '[REDACTED]');
@@ -144,6 +152,38 @@ export function redactSecrets(message: string): string {
   return out;
 }
 
+// Every ESC-initiated terminal sequence — not just the colour/style CSI subset
+// ANSI_PATTERN strips. Alternatives, tried in order at each ESC:
+//   1. CSI with parameter (incl. private-mode `?`), intermediate, and final
+//      bytes per ECMA-48 — catches \x1b[?25l (hide cursor) and kin.
+//   2. OSC to BEL or ST (or truncated) — catches \x1b]0;title\x07 retitling.
+//   3. DCS/SOS/PM/APC to ST (or truncated).
+//   4. Bare ESC plus at most one byte — the fallback that also swallows a
+//      trailing/truncated ESC rather than leaking it.
+// (Built from strings, so no-control-regex has nothing to flag here.)
+const TERMINAL_ESCAPE_PATTERN = new RegExp(
+  '\\x1b\\[[0-?]*[ -/]*[@-~]?' +
+  '|\\x1b\\][^\\x07\\x1b]*(?:\\x07|\\x1b\\\\)?' +
+  '|\\x1b[PX^_][^\\x1b]*(?:\\x1b\\\\)?' +
+  '|\\x1b.?',
+  'g',
+);
+
+/**
+ * Strip terminal escape sequences and residual control bytes from a line bound
+ * for the CONSOLE sink. redactSecrets' ANSI_PATTERN removes only plain CSI
+ * colour/style sequences, so private-mode CSI (\x1b[?25l), OSC window-retitling
+ * (\x1b]0;…\x07), and DCS/SOS/PM/APC embedded in untrusted content would
+ * otherwise reach the user's terminal verbatim under PI_RESEARCH_CONSOLE_LOG.
+ * The console sink keeps its own explicit CR/LF→"" replace (the form CodeQL
+ * recognizes as the log-injection sanitizer); this handles everything else,
+ * spacing out any leftover C0/DEL byte (BEL, backspace, …) like
+ * neutralizeControlChars does.
+ */
+export function stripTerminalEscapes(message: string): string {
+  return message.replace(TERMINAL_ESCAPE_PATTERN, '').replace(CONTROL_CHARS_PATTERN, ' ');
+}
+
 /**
  * Neutralize control characters (newlines, carriage returns, etc.) so that
  * untrusted content cannot forge or corrupt a raw console log line

package/src/utils/metrics-summary.ts

@@ -137,9 +137,17 @@ export function extractRunStats(snapshot: IMetricsSnapshot): ResearchStats | nul
   const browserSearchQueries = sumCounter(counters, 'browser_search_queries_total');
   const searchQueries = sumCounter(counters, 'search_queries_total') + browserSearchQueries;
 
-  // URLs discovered via search
-  const urlsDiscovered = sumCounter(counters, 'browser_search_results_total') +
-    sumCounter(counters, 'browser_search_unique_urls_total');
+  // URLs discovered via search. browser_search_unique_urls_total is the
+  // cross-query deduped subset of the SAME batch counted by
+  // browser_search_results_total (raw per-query results), so summing both
+  // near-doubles the figure. The unique counter IS the discovered count; the
+  // raw counter is only a fallback for a search that threw before completing
+  // the batch (the unique count is emitted after cross-query dedup, so such a
+  // run never emits it).
+  const uniqueUrlsDiscovered = sumCounter(counters, 'browser_search_unique_urls_total');
+  const urlsDiscovered = uniqueUrlsDiscovered > 0
+    ? uniqueUrlsDiscovered
+    : sumCounter(counters, 'browser_search_results_total');
 
   // URLs scraped — outcomes from scrape_results_total
   const fetchSuccess = getLabeledCounter(counters, 'scrape_results_total', { outcome: 'fetch_success' });
@@ -178,12 +186,22 @@ export function extractRunStats(snapshot: IMetricsSnapshot): ResearchStats | nul
   // Cost — sum all label variants
   const cost = sumCounter(counters, 'llm_cost_total');
 
-  // Duration from histogram
-  const durationMs = histograms['research_session_duration_ms{mode="deep",complexity="1",status="success"}']?.max ||
-    histograms['research_session_duration_ms{mode="deep",complexity="2",status="success"}']?.max ||
-    histograms['research_session_duration_ms{mode="deep",complexity="3",status="success"}']?.max ||
-    histograms['research_session_duration_ms{mode="quick",complexity="0",status="success"}']?.max ||
-    0;
+  // Duration from histogram. The registry serializes label keys in ALPHABETICAL
+  // order (MetricsRegistry.serializeLabels), so the real key is
+  // `{complexity="…",mode="…",status="…"}` — a lookup that hardcodes any other
+  // label order can never match. Complexity is also emitted as String(complexity)
+  // with no fixed value set. Scan by metric-name prefix + success status instead,
+  // taking the largest observed max across matching label variants.
+  let durationMs = 0;
+  for (const key of Object.keys(histograms)) {
+    if (
+      key.startsWith('research_session_duration_ms{') &&
+      key.includes('status="success"')
+    ) {
+      const max = histograms[key]?.max ?? 0;
+      if (max > durationMs) durationMs = max;
+    }
+  }
 
   // Rounds completed — evaluator runs give us the round count
   const roundsCompleted = sumCounter(counters, 'evaluator_runs_total');

package/src/utils/stdio-capture.ts

@@ -33,6 +33,26 @@ const sessionCaptureStates = new Map<string, boolean>();
 let unkeyedCaptureDepth = 0;
 let isAnyLoggerCapturingOutput = false;
 
+/**
+ * Active captures in creation order. Restoration MUST be LIFO: each capture saves
+ * the currently-installed writers/console/fs.writeSync (and the current FD-2
+ * target) as its "originals", so a frame that finishes while a later frame is
+ * still active must NOT restore — it would re-install the still-active frame's
+ * view of the world later, when that frame finally exits, re-pointing stderr,
+ * stdout, console.* and OS-level FD 2 at a dead capture's log-file plumbing for
+ * the remainder of the process. Keyed captures nest by design (a run's capture
+ * around the embedding server's), and nothing orders their exits — a cancelled
+ * run's outer capture routinely ends while the embedder-init capture is mid
+ * flight. So: an out-of-order finisher only marks its frame retired; the frame
+ * on top of the stack performs the actual restores, unwinding every already-
+ * retired frame beneath it in strict reverse order.
+ */
+interface CaptureFrame {
+  retired: boolean;
+  restore: () => void;
+}
+const captureStack: CaptureFrame[] = [];
+
 /**
  * Recompute the global "is anything capturing" flag from BOTH the keyed sessions and
  * the unkeyed-capture depth, so neither scheme can clobber the other's state.
@@ -220,7 +240,9 @@ export async function captureStdio<T>(
       if (shouldRedirectStderr(stat)) {
         savedFd2 = fs.openSync('/dev/fd/2', 'a');
         fs.closeSync(2);
-        const newFd = fs.openSync(logFile, 'a');
+        // 0o600 applies only when this open CREATES the file — keeps the log
+        // owner-only in a world-traversable tmpdir if rotation removed it.
+        const newFd = fs.openSync(logFile, 'a', 0o600);
         if (newFd === 2) {
           fd2Redirected = true;
         } else {
@@ -371,20 +393,41 @@ export async function captureStdio<T>(
     }
   } catch (_e) { /* ignore */ }
 
+  const frame: CaptureFrame = {
+    retired: false,
+    restore: () => {
+      process.stderr.write = originalStderrWrite;
+      process.stdout.write = originalStdoutWrite;
+      try {
+        fs.writeSync = originalFsWriteSync;
+      } catch { /* ignore */ }
+      console.log = originalConsole.log;
+      console.info = originalConsole.info;
+      console.warn = originalConsole.warn;
+      console.error = originalConsole.error;
+      console.debug = originalConsole.debug;
+
+      // Restore FD 2 to what it was when THIS frame started (for a nested frame
+      // that is the enclosing frame's log-file redirect; the enclosing frame's
+      // own restore, which always runs after this one in the unwind, then puts
+      // the true terminal FD back).
+      if (fd2Redirected && savedFd2 >= 0) {
+        try {
+          fs.closeSync(2);
+          const r = fs.openSync(`/dev/fd/${savedFd2}`, 'a');
+          if (r !== 2 && r >= 0) {
+            // Didn't recover FD 2 — leave restored FD open
+          }
+          try { fs.closeSync(savedFd2); } catch { /* ignore */ }
+        } catch { /* ignore restore errors */ }
+      }
+    },
+  };
+  captureStack.push(frame);
+
   try {
     return await task();
   } finally {
-    // Restore everything
-    process.stderr.write = originalStderrWrite;
-    process.stdout.write = originalStdoutWrite;
-    try {
-      fs.writeSync = originalFsWriteSync;
-    } catch { /* ignore */ }
-    console.log = originalConsole.log;
-    console.info = originalConsole.info;
-    console.warn = originalConsole.warn;
-    console.error = originalConsole.error;
-    console.debug = originalConsole.debug;
     if (sessionId) {
       setSessionCapturing(sessionId, false);
     } else {
@@ -392,16 +435,14 @@ export async function captureStdio<T>(
       recomputeGlobalCapturing();
     }
 
-    // Restore FD 2
-    if (fd2Redirected && savedFd2 >= 0) {
-      try {
-        fs.closeSync(2);
-        const r = fs.openSync(`/dev/fd/${savedFd2}`, 'a');
-        if (r !== 2 && r >= 0) {
-          // Didn't recover FD 2 — leave restored FD open
-        }
-        try { fs.closeSync(savedFd2); } catch { /* ignore */ }
-      } catch { /* ignore restore errors */ }
+    // LIFO unwind (see captureStack): only the top frame restores; an
+    // out-of-order finisher parks as retired and is unwound — in reverse
+    // order — when everything above it has finished. Until then its patches
+    // stay installed, which merely extends capture a little; restoring early
+    // would divert stdio permanently.
+    frame.retired = true;
+    while (captureStack.length > 0 && captureStack[captureStack.length - 1]!.retired) {
+      captureStack.pop()!.restore();
     }
   }
 }

package/src/utils/text-utils.ts

@@ -76,7 +76,14 @@ export function truncateWithMarker(content: string, maxChars: number): string {
   const makeMarker = (kept: number) => `\n\n[content truncated: showing ${kept} of ${total} chars]`;
   // Reserve marker room using the longest possible marker (kept ≤ total, so the
   // digit count of `kept` never exceeds that of `total`).
-  const keep = Math.max(0, maxChars - makeMarker(total).length);
+  let keep = Math.max(0, maxChars - makeMarker(total).length);
+  // slice() counts UTF-16 code units, so the cut can land between the halves of a
+  // surrogate pair — a lone high surrogate in the body is invalid text that some
+  // providers reject (400) when it reaches a JSON payload. Backing off one unit
+  // restores well-formed text. The whitespace-boundary path below cannot split a
+  // pair: it cuts immediately before a BMP whitespace character.
+  const lastKeptUnit = content.charCodeAt(keep - 1);
+  if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
   let body = content.slice(0, keep);
   // Prefer a clean boundary near the cut point.
   const boundary = Math.max(body.lastIndexOf('\n'), body.lastIndexOf(' '));

package/src/utils/tool-usage-tracker.ts

@@ -125,13 +125,20 @@ export class ToolUsageTracker {
     
     const toolLimit = this.limits[toolName as keyof ToolLimits];
     const catLimit = usage.limit;
+    // Decide by which limit actually tripped, computed exactly as recordCall
+    // computes it: a first-ever `search` blocked by the shared gathering cap must
+    // NOT claim "you have already used your one search call" — the agent would
+    // wrongly believe it had searched. The per-tool wording is only correct when
+    // the per-tool counter itself is at its limit.
+    const toolCount = usage.toolCounts.get(toolName) || 0;
+    const toolLimitTripped = toolLimit !== undefined && toolCount >= toolLimit;
 
-    if (toolName === 'search' && toolLimit === 1) {
+    if (toolName === 'search' && toolLimit === 1 && toolLimitTripped) {
         return `SEARCH LIMIT REACHED: You have already used your one search call. ` +
             `Proceed to scraping: use the scrape tool for full deep-dives into your best search results.`;
     }
 
-    if (toolName === 'youtube_transcript' && toolLimit === 1) {
+    if (toolName === 'youtube_transcript' && toolLimit === 1 && toolLimitTripped) {
         return `YOUTUBE TRANSCRIPT LIMIT REACHED: You have already used your one youtube_transcript call. ` +
             `Continue with your other gathering tools (search, scrape) and proceed to synthesis.`;
     }

package/src/web-research/browser-search.ts

@@ -27,14 +27,22 @@ export async function performSearch(
     onProgress?: (links: number) => void,
     container: ServiceContainer = getServiceContainer(),
     /**
-     * Optional sink for per-query FAILURES (timeout / worker error). Populated as
-     * an out-param rather than folded into the return value because the returned
-     * map's `query -> results` shape is what every caller consumes; a query absent
-     * from this map produced zero results with no failure, i.e. the search really
-     * did come back empty. Without this the caller cannot tell those apart and
-     * reports every empty query as a too-narrow query.
+     * Optional sink for per-query FAILURES (timeout / worker error / cross-query
+     * dedup). Populated as an out-param rather than folded into the return value
+     * because the returned map's `query -> results` shape is what every caller
+     * consumes; a query absent from this map produced zero results with no
+     * failure, i.e. the search really did come back empty. Without this the
+     * caller cannot tell those apart and reports every empty query as a
+     * too-narrow query.
      */
     failures?: Map<string, QueryFailure>,
+    /**
+     * Session/research id keying the per-session circuit breaker inside
+     * runWorkerSearch — mirroring the scrape path, which threads it via
+     * BrowserTask.sessionId. Callers without a session (undefined) share the
+     * global breaker, as before.
+     */
+    sessionId?: string,
 ): Promise<Map<string, SearchResult[]>> {
     const startTime = Date.now();
     const resultMap = new Map<string, SearchResult[]>();
@@ -116,7 +124,7 @@ export async function performSearch(
               ? AbortSignal.any([signal, timeoutController.signal])
               : timeoutController.signal;
 
-            const results = await runWorkerSearch(query, config, querySignal, 1, undefined, container);
+            const results = await runWorkerSearch(query, config, querySignal, 1, sessionId, container);
             const queryDuration = Date.now() - queryStartTime;
             metrics.observe('browser_search_query_duration_ms', queryDuration);
 
@@ -142,6 +150,17 @@ export async function performSearch(
                         uniqueResults.push(r);
                     }
                 }
+                if (uniqueResults.length === 0) {
+                    // Every raw result deduplicated away against URLs earlier
+                    // queries in this batch already returned. Without an entry
+                    // here the query fell through to the empty_results default —
+                    // "the query may be too narrow" — sending the researcher off
+                    // rewriting a query that actually worked.
+                    failures?.set(query, {
+                        type: 'all_duplicates',
+                        message: `All ${results.length} results for this query duplicated URLs already returned by earlier queries in this batch. The query succeeded but surfaced nothing new — do not rewrite it; try a different angle only if you still need additional sources.`,
+                    });
+                }
                 resultMap.set(query, uniqueResults);
             } else {
                 metrics.increment('browser_search_queries_total', 1, { status: 'no_results' });

package/src/web-research/scraper-utils.ts

@@ -343,6 +343,11 @@ function isPrivateIpv6(ip: string): boolean {
   // Link-local (fe80::/10)
   if (normalized.startsWith('fe8') || normalized.startsWith('fe9') ||
       normalized.startsWith('fea') || normalized.startsWith('feb')) return true;
+  // Deprecated site-local (fec0::/10, RFC 3879, deprecated 2004). No OS
+  // auto-routes to it today, but it's still squarely "private/reserved"
+  // address space — the sibling range immediately above this one.
+  if (normalized.startsWith('fec') || normalized.startsWith('fed') ||
+      normalized.startsWith('fee') || normalized.startsWith('fef')) return true;
   // IPv4-mapped ::ffff:x.x.x.x — check the embedded IPv4 part. The mapped tail
   // can arrive dotted-decimal (::ffff:127.0.0.1) OR hex (::ffff:7f00:1) — Node's
   // URL parser normalizes literals to the hex form — and the prefix can be
@@ -357,6 +362,14 @@ function isPrivateIpv6(ip: string): boolean {
   // legitimate scrape target, so block the whole ranges rather than decode them.
   if (normalized.startsWith('2002:')) return true;
   if (/^2001:0{0,4}:/.test(normalized) || normalized.startsWith('2001::')) return true;
+  // NAT64 well-known prefix (64:ff9b::/96, RFC 6052) also tunnels an IPv4
+  // address, and unlike the transitional ranges above it's actively deployed
+  // today (464XLAT on IPv6-only mobile/enterprise networks). A host whose
+  // outbound path traverses a NAT64 gateway gets this transparently
+  // translated to a direct connection to the embedded IPv4 (e.g. the cloud
+  // metadata service) — block the whole prefix regardless of what it wraps,
+  // same rationale as the 6to4/Teredo ranges just above.
+  if (normalized.startsWith('64:ff9b:')) return true;
   // All-zeros (unspecified address)
   if (normalized === '::' || normalized === '0:0:0:0:0:0:0:0') return true;
   return false;

package/src/web-research/search.ts

@@ -23,6 +23,9 @@ import type { ServiceContainer } from '../core/service-registry.ts';
  * @param config - Optional configuration override
  * @param signal - Optional AbortSignal
  * @param onProgress - Optional callback with cumulative link count across completed queries
+ * @param container - Service container for scheduler resolution
+ * @param sessionId - Research/session id keying the per-session circuit breaker
+ *   (mirrors the scrape path); undefined callers share the global breaker
  * @returns Promise<QueryResultWithError[]> - Array of search results with error information
  */
 export async function search(
@@ -30,7 +33,8 @@ export async function search(
   config?: Config,
   signal?: AbortSignal,
   onProgress?: (links: number) => void,
-  container: ServiceContainer = getServiceContainer()
+  container: ServiceContainer = getServiceContainer(),
+  sessionId?: string
 ): Promise<QueryResultWithError[]> {
   if (queries.length === 0) return [];
   
@@ -42,7 +46,7 @@ export async function search(
     // Per-query failures (timeout / dead worker) come back separately from the
     // results so an empty result set can be attributed correctly below.
     const failures = new Map<string, QueryFailure>();
-    const resultMap = await performSearch(queries, config, signal, onProgress, container, failures);
+    const resultMap = await performSearch(queries, config, signal, onProgress, container, failures, sessionId);
     const searchDuration = Date.now() - searchStart;
     metrics.observe('search_latency_ms', searchDuration);
     

package/src/web-research/types.ts

@@ -24,10 +24,12 @@ export interface QueryResultWithError {
  * at all about the query and should prompt a retry instead. Collapsing the two —
  * reporting every zero-result query as "the query may be too narrow" — sent the
  * agent off rewriting perfectly good queries whenever the browser pool was the
- * thing that failed.
+ * thing that failed. `all_duplicates` is the third case: the query SUCCEEDED,
+ * but every result deduplicated away against URLs earlier queries in the batch
+ * already returned — coverage feedback, not query-quality feedback.
  */
 export interface QueryFailure {
-  type: 'empty_results' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
+  type: 'empty_results' | 'all_duplicates' | 'service_unavailable' | 'timeout' | 'network_error' | 'unknown';
   message: string;
 }
 

package/src/web-research/web-scraper.ts

@@ -80,7 +80,11 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
     const sizeMB = Math.round(bytes.length / 1024 / 1024);
     logger.warn(`[Scrapers] PDF too large (${sizeMB}MB, max 100MB), skipping extraction`);
     metrics.increment('scrape_pdf_errors_total', 1, { error_type: 'size_exceeded' });
-    return `*Error: PDF too large (${sizeMB}MB, max 100MB).*`;
+    // THROW, never an in-band "*Error: ...*" string: a returned banner flowed
+    // into validateContent as if it were page content, and a verbose one
+    // (≥50 words / ≥200 chars) cleared the stub gate — the failure banner was
+    // then cached and cited as a successful scrape.
+    throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
   }
 
   const pdfExtractionStart = Date.now();
@@ -114,7 +118,9 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
       contentType: 'pdf',
       errorType: 'extraction_failed',
     });
-    return `*Error: Could not extract content from PDF (${msg}).*`;
+    // See the size branch above: an extraction failure must FAIL the scrape,
+    // not travel in-band as pseudo-content.
+    throw new Error(`Could not extract content from PDF (${msg})`, { cause: e });
   }
 }
 
@@ -354,8 +360,30 @@ async function scrapeWithStealthBrowser(_url: string, config?: Config, signal?:
     const result = await runBrowserTask<any>({ url: _url, sessionId }, 'scrape', config, signal, 1, container);
     const browserDuration = Date.now() - browserStart;
 
-    if (result.buffer) {
-      const markdown = await extractPdfToMarkdown(new Uint8Array(result.buffer));
+    // PDF bytes travel base64-encoded (bufferB64): the worker result crosses the
+    // cluster-IPC JSON hop (and, for followers, a second JSON hop through the
+    // leader's browser-server), which turns a raw Buffer into
+    // {type:'Buffer',data:[...]} — new Uint8Array(that) is ZERO bytes, so the
+    // extraction ran on nothing and its error string was cached as a "success".
+    // The JSON-roundtripped legacy shape is still decoded so a mixed-version
+    // leader/follower pair keeps working.
+    const legacyBuffer = result.buffer;
+    const pdfBytes: Buffer | null =
+      typeof result.bufferB64 === 'string'
+        ? Buffer.from(result.bufferB64, 'base64')
+        : Array.isArray(legacyBuffer?.data)
+          ? Buffer.from(legacyBuffer.data)
+          : legacyBuffer instanceof Uint8Array
+            ? Buffer.from(legacyBuffer)
+            : null;
+
+    if (pdfBytes !== null) {
+      const markdown = await extractPdfToMarkdown(pdfBytes);
+      // Mirror the fetch layer's PDF branch: extraction failures THROW from
+      // extractPdfToMarkdown, and validateContent still fails a nominally
+      // successful extraction that produced no real content — neither may be
+      // recorded (and cached) as a success.
+      validateContent('', markdown, _url);
       metrics.increment('scrape_operations_total', 1, { layer: 'playwright', content_type: 'pdf', status: 'success' });
       metrics.observe('scrape_latency_ms', browserDuration, { layer: 'playwright', content_type: 'pdf', status: 'success' });
       return { source: 'playwright', layer: 'playwright+camoufox', markdown };
@@ -386,6 +414,10 @@ async function scrapeWithStealthBrowser(_url: string, config?: Config, signal?:
     metrics.observe('scrape_latency_ms', browserDuration, { layer: 'playwright', content_type: 'html', status: 'success' });
     return { source: 'playwright', layer: 'playwright+camoufox', markdown };
   } catch (error) {
+    // A caller cancellation is not a scrape failure: no layer error metrics and
+    // no tracker entry — the search path applies the same 'Aborted' special case
+    // end-to-end (see browser-search.ts). scrapeSingle classifies it above us.
+    if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw error;
     const browserDuration = Date.now() - browserStart;
     metrics.increment('scrape_operations_total', 1, { layer: 'playwright', status: 'error' });
     metrics.observe('scrape_latency_ms', browserDuration, { layer: 'playwright', status: 'error' });
@@ -401,6 +433,15 @@ async function scrapeWithStealthBrowser(_url: string, config?: Config, signal?:
 }
 
 export async function scrapeSingle(url: string, signal?: AbortSignal, config?: Config, sessionId?: string, container: ServiceContainer = getServiceContainer()): Promise<ScrapeResult> {
+  // Cancellation short-circuit: after a user abort each remaining URL would
+  // otherwise still pay for SSRF DNS validation, a doomed fetch, and a browser
+  // fallback — all logged/counted as failures the user never had. 'Aborted' is
+  // the codebase's recognized cancellation signature (see browser-search.ts);
+  // deliberately no total_failure tally, no tracker entry, no ERROR log.
+  if (signal?.aborted) {
+    return { url, success: false, error: 'Aborted', markdown: '' };
+  }
+
   // Guard against an array accidentally stringified and passed as a single url (JSON arrays start
   // with '['). Do NOT reject brackets ANYWHERE: valid URLs legitimately contain them in query
   // strings (e.g. ?ids[]=1) and IPv6 literals (http://[::1]/) — the old includes('[') check dropped
@@ -477,6 +518,13 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
 
     return { ...res, url, success: true };
   } catch (e1) {
+    // Cancelled mid-fetch: not a fetch-layer failure, and no reason to pay for
+    // the browser fallback — classify as the recognized cancellation (no ERROR
+    // log, no tracker entry, no total_failure tally).
+    if (signal?.aborted || (e1 instanceof Error && e1.message === 'Aborted')) {
+      logger.debug(`[Scrapers] Scrape cancelled for ${url}`);
+      return { url, success: false, error: 'Aborted', markdown: '' };
+    }
     const fetchDuration = Date.now() - start;
     logger.debug(`[Scrapers] fetch failed for ${url} in ${fetchDuration}ms: ${formatErrorWithCause(e1)}`);
     // Tracked here rather than inside scrapeWithFetch, which is retried above —
@@ -499,6 +547,14 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
 
         return { ...res, url, success: true };
       } catch (e2) {
+        // See the fetch-layer catch above: a cancellation surfacing from the
+        // browser layer (runBrowserTask throws exactly 'Aborted' on a caller
+        // abort) is not a failure — it must not reach the ERROR log, the error
+        // tracker, or the total_failure tally.
+        if (signal?.aborted || (e2 instanceof Error && e2.message === 'Aborted')) {
+          logger.debug(`[Scrapers] Browser scrape cancelled for ${url}`);
+          return { url, success: false, error: 'Aborted', markdown: '' };
+        }
         const totalDuration = Date.now() - start;
         // A blocked/stubbed/HTTP-errored/slow/pool-draining URL is an expected
         // scraping outcome, already surfaced to the user as "not scraped" (the
@@ -530,6 +586,11 @@ export async function scrape(urls: string[], maxConcurrency = 5, signal?: AbortS
   
   const results: ScrapeResult[] = [];
   for (let i = 0; i < urls.length; i += maxConcurrency) {
+    // Stop dispatching once the caller has cancelled; URLs already in flight
+    // settle via scrapeSingle's own cancellation classification. Without this,
+    // every remaining batch still ran its full fetch + browser-fallback cycle
+    // after a user cancel.
+    if (signal?.aborted) break;
     const batch = urls.slice(i, i + maxConcurrency);
     const batchRes = await Promise.all(
       batch.map(async url => {