@lincoln504/pi-research 1.0.10 → 1.1.0

package/src/orchestration/research-orchestration-service.ts

@@ -37,6 +37,7 @@ import type { Config } from '../config.ts';
 import { getConfig, DEFAULTS } from '../config.ts';
 import { getCachedScrapedContent, normalizeUrl, cleanupSharedLinks } from '../utils/shared-links.ts';
 import { runResearcher } from './researcher-executor.ts';
+import { ResearchRunSemaphore, ResearchRunCapacityError } from '../infrastructure/research-run-semaphore.ts';
 import { recordResearcherFailure, shouldStopResearch, createResearchStopError } from './session-state.ts';
 import type { ResearchSessionService } from './research-session-service.ts';
 import { QuickResearchOrchestrator } from './quick-research-orchestrator.ts';
@@ -107,6 +108,36 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
 
     const researchStart = Date.now();
 
+    // Cross-process run-cap: acquire one of N slots so concurrent research runs
+    // can't saturate the shared leader-elected browser/embedding pool and degrade
+    // into ~60-error failures. Over-cap runs QUEUE for a slot (announced once via
+    // the observer) rather than failing, because an extra run is a legitimate
+    // request to do work, not an error. Fail-OPEN on internal/IO errors (a
+    // semaphore bug must never break research); only a genuine
+    // ResearchRunCapacityError — nothing freed within the whole queue window —
+    // propagates. The slot is released in finally.
+    let runSlot: { slotIndex: number; release(): Promise<void> } | null = null;
+    try {
+      const container = tryGetServiceContainerFromCtx(ctx);
+      const semaphore = await getService<ResearchRunSemaphore>(ServiceNames.RESEARCH_RUN_SEMAPHORE, ctx, container);
+      runSlot = await semaphore.acquire(undefined, signal, (slots, maxWaitMs) => {
+        // The queue notice fires BEFORE any orchestrator exists, so `observer` is
+        // still exactly what the caller passed: either a ResearchObserver, or the
+        // HeadlessObserverOptions bag (only `onProgress`) that the CLI supplies and
+        // the orchestrator would normally wrap later. Dispatch to whichever shape
+        // is present — handling only the former silently drops the notice on the
+        // CLI, which is the very front-end that most needs it.
+        notifyRunQueued(observer, slots, maxWaitMs);
+      });
+      logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
+    } catch (err) {
+      if (err instanceof ResearchRunCapacityError) throw err; // capacity exhausted → fail fast
+      // A cancel while queueing for a slot must stay cancelled. Fail-open here would
+      // start the very run the user just aborted.
+      if (err instanceof Error && err.name === 'AbortError') throw err;
+      logger.warn(`[ResearchOrchestrationService] Run-cap unavailable, proceeding without it: ${err instanceof Error ? err.message : String(err)}`);
+    }
+
     let result: string;
     try {
       if (depth === 0) {
@@ -149,6 +180,20 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
       metrics.observe('research_manager_latency_ms', researchDuration, { depth: String(depth), status: 'error', source: 'extension' });
       metrics.increment('research_manager_requests_total', 1, { depth: String(depth), status: 'error', source: 'extension' });
       throw error;
+    } finally {
+      // Always release the run-cap slot — on success, error, or abort (the
+      // orchestrator throws on AbortSignal, hitting this finally before rethrow).
+      if (runSlot) {
+        // Never let a release failure mask the run's own outcome — but do report it.
+        // A slot that fails to release stays held for the lifetime of THIS process
+        // (liveness reclaim only fires once the owner is gone), so it silently costs
+        // one unit of capacity for every subsequent run in the session.
+        await runSlot.release().catch((relErr) => {
+          logger.warn(
+            `[ResearchOrchestrationService] Failed to release run slot ${runSlot?.slotIndex}; capacity is reduced until this process exits: ${relErr instanceof Error ? relErr.message : String(relErr)}`,
+          );
+        });
+      }
     }
   }
 
@@ -548,3 +593,28 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
     }
   }
 }
+
+/**
+ * Deliver the "your run is queued behind other runs" notice to whichever observer
+ * shape the caller supplied.
+ *
+ * This fires before the orchestrator normalizes the observer, so both forms are
+ * live here: a {@link ResearchObserver} instance (SDK / pi extension) or the
+ * {@link HeadlessObserverOptions} bag carrying only `onProgress` (the CLI and
+ * agent skill). A queued run is otherwise completely silent for the whole acquire
+ * wait, which is indistinguishable from a hang.
+ *
+ * Never throws: a front-end callback must not be able to fail the run it is
+ * merely reporting on.
+ */
+function notifyRunQueued(observer: unknown, slots: number, maxWaitMs: number): void {
+  try {
+    const obs = observer as
+      | { onRunQueued?: (s: number, w: number) => void; onProgress?: (e: string, d?: unknown) => void }
+      | undefined;
+    if (typeof obs?.onRunQueued === 'function') obs.onRunQueued(slots, maxWaitMs);
+    else if (typeof obs?.onProgress === 'function') obs.onProgress('run_queued', { slots, maxWaitMs });
+  } catch (err) {
+    logger.debug(`[ResearchOrchestrationService] run_queued notification threw: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}

package/src/orchestration/research-synthesis-service.ts

@@ -13,6 +13,7 @@ import { parseCitations } from '../utils/text-utils.ts';
 import { normalizeCitations, formatCitedLinks, type GlobalCitation } from '../utils/citation-utils.ts';
 import { lastCitedLinksHeaderIndex } from '../utils/text-utils.ts';
 import { getScrapedLinks, isTranscribedLink, normalizeUrl } from '../utils/shared-links.ts';
+import { stripTrailingLlmPunctuation } from '../utils/url-utils.ts';
 import { logger } from '../logger.ts';
 import { ServiceLifecycle, type IService } from '../core/service-registry.ts';
 import { ServiceNames } from '../core/interfaces/service-names.ts';
@@ -26,6 +27,125 @@ import type { SteeringMessage } from './session-state.ts';
 const NO_SOURCES_NOTE =
   'CITED LINKS\n_No web sources were successfully retrieved for this report — searches or page fetches may have been blocked (for example by site bot-protection) or returned no usable results. Treat the findings above as unverified._';
 
+// Appended when the model wrote its own links but the run retrieved nothing we could
+// verify them against. The block is kept (it may still be a useful lead) but must not
+// read as though the engine checked it.
+const UNVERIFIED_LINKS_NOTE =
+  '_None of the links above could be verified: no page fetch succeeded this session and no citation could be matched to retrieved content. They may be inaccurate or invented — check each one before relying on it._';
+
+/**
+ * Largest `[N]` treated as a citation marker. Inline markers index a source list
+ * that is realistically well under this; the bound exists so an ordinary bracketed
+ * number in prose — a year (`[2023]`), an issue id, an array index — is left alone
+ * instead of being silently deleted as a "dangling citation".
+ */
+const MAX_PLAUSIBLE_CITATION_MARKER = 200;
+
+/**
+ * Rewrite inline `[N]` citation markers in a prose body: renumber the ones we can
+ * map, delete the ones known to dangle, leave everything else untouched.
+ *
+ * Deletion is whitespace-aware rather than followed by a global space collapse.
+ * A blanket `text.replace(/  +/g, ' ')` also eats *leading* indentation, which
+ * silently flattens nested list levels into one and turns 4-space indented code
+ * blocks into paragraphs — corrupting the structure of every report that contains
+ * them. So the pad preceding a deleted marker is consumed only when the marker is
+ * mid-line (i.e. real prose spacing); line-leading indentation is preserved.
+ *
+ * Code is skipped entirely. A report may quote code containing bracketed integers
+ * (`items[12]`), and on the no-CITED-LINKS path *every* out-of-range marker is treated
+ * as dangling — so without this guard a quoted array index would be silently deleted
+ * from the user's code sample. Fenced blocks are tracked across lines; single-backtick
+ * spans are detected by an odd backtick count earlier on the same line.
+ *
+ * @param remap  local marker number → verified global id
+ * @param isDangling  whether a marker not present in `remap` should be deleted
+ */
+function rewriteCitationMarkers(
+  text: string,
+  remap: Map<number, number>,
+  isDangling: (n: number) => boolean,
+): string {
+  const inFence = fencedCodeMask(text);
+  return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad: string, digits: string, offset: number) => {
+    const n = parseInt(digits, 10);
+    const mapped = remap.get(n);
+    if (mapped !== undefined) return `${pad}[${mapped}]`;
+    // Citation numbering is 1-based; `[0]` is always an index or a literal, never a marker.
+    if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
+    if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
+    // Preserve indentation when nothing but whitespace precedes on this line;
+    // otherwise drop the pad too so no double space is left behind.
+    const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
+    return text.slice(lineStart, offset).trim() === '' ? pad : '';
+  });
+}
+
+/**
+ * Build an `offset → is inside a ``` fenced block` predicate for `text`.
+ *
+ * Computed once per rewrite (a single line scan) rather than per marker, so marker
+ * rewriting stays linear in the body length.
+ */
+function fencedCodeMask(text: string): (offset: number) => boolean {
+  const ranges: Array<[number, number]> = [];
+  let open = -1;
+  let pos = 0;
+  for (const line of text.split('\n')) {
+    if (/^\s{0,3}(```|~~~)/.test(line)) {
+      if (open < 0) open = pos;
+      else {
+        ranges.push([open, pos + line.length]);
+        open = -1;
+      }
+    }
+    pos += line.length + 1; // +1 for the '\n' consumed by split
+  }
+  // An unterminated fence runs to the end of the body — treat the remainder as code.
+  if (open >= 0) ranges.push([open, text.length]);
+  return (offset: number) => ranges.some(([start, end]) => offset >= start && offset < end);
+}
+
+/** Whether `offset` sits inside a single-backtick code span on its own line. */
+function inInlineCodeSpan(text: string, offset: number): boolean {
+  const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
+  let ticks = 0;
+  for (let i = lineStart; i < offset; i++) if (text[i] === '`') ticks++;
+  return ticks % 2 === 1;
+}
+
+/**
+ * Redact inline `https?://` tokens in synthesis prose whose URL was NOT retrieved
+ * this session (absent from `trusted`, a set of normalized verified-source URLs).
+ *
+ * The synthesis body is otherwise passed through verbatim, so an unverified inline
+ * URL is the one channel by which a fabricated source can reach the user — and
+ * inline URLs violate the required format (links belong in CITED LINKS only).
+ * Legitimate inline references to real sources are preserved (they are present in
+ * `trusted`). Returns the (possibly) redacted text.
+ */
+function redactUnverifiedProseUrls(text: string, trusted: Set<string>): string {
+  if (trusted.size === 0) return text;
+  let removed = 0;
+  // Brackets are NOT excluded from the token: a bare `[^\s)\]>"']+` truncates
+  // `.../Python_(programming_language)` at the first `)`, so the truncated form
+  // never matches its own entry in `trusted` and a perfectly good Wikipedia link
+  // gets redacted. Instead take the greedy token and split off exactly the
+  // trailing punctuation `normalizeUrl` itself discards — so the compared string
+  // and the redacted span always agree — restoring that tail afterwards.
+  const result = text.replace(/https?:\/\/[^\s<>"']+/gi, (token) => {
+    const core = stripTrailingLlmPunctuation(token);
+    const tail = token.slice(core.length);
+    if (!core || trusted.has(normalizeUrl(core))) return token;
+    removed++;
+    return `[link removed: not found in retrieved sources]${tail}`;
+  });
+  if (removed > 0) {
+    logger.warn(`[ResearchSynthesisService] Redacted ${removed} unverified inline URL(s) from synthesis prose (not present in retrieved sources).`);
+  }
+  return result;
+}
+
 /**
  * Research Synthesis Service
  *
@@ -178,6 +298,18 @@ export class ResearchSynthesisService implements IService {
     let globalCitations: GlobalCitation[] =
       reports.size > 0 ? normalizeCitations(reports).globalCitations : [];
 
+    // NOTE: a "provenance gate" previously DROPPED citations tagged Scrape/Transcript
+    // whose URL was absent from the session's successful-scrape pool. It was REMOVED
+    // because it conflated a failed/blocked scrape (real URL, content unretrieved) with
+    // fabrication: under the common high scrape-failure rate (bot blocks on GitHub /
+    // Microsoft / etc.) it mass-dropped legitimate citations, and the remap below then
+    // neutralized their inline [N] markers into a broken '[removed]' token that reached
+    // users. We now KEEP every parsed citation (deduped by URL) and never emit
+    // '[removed]'; a dangling inline marker is removed cleanly instead. Fabrication
+    // defense rests on the URL dedup + the prose-URL sweep, which do not punish failed
+    // scrapes. (The successful-scrape pool is still reconciled below to FOLD IN any
+    // sources the researchers scraped but failed to cite.)
+
     // Robustness fallback: reconcile against ground-truth provenance — the URLs
     // ACTUALLY, successfully scraped this session (registerScrapedLinks only records
     // HTTP-success fetches). This runs even when SOME citations parsed: a
@@ -216,6 +348,7 @@ export class ResearchSynthesisService implements IService {
 
     if (globalCitations.length > 0) {
       const verifiedLinksSection = formatCitedLinks(globalCitations);
+      const trustedUrls = new Set(globalCitations.map((c) => normalizeUrl(c.url)));
       // Replace an existing CITED LINKS section with the verified version — but only
       // a line-leading, all-caps "CITED LINKS" header marks the section. An
       // unanchored /CITED LINKS[\s\S]*$/i would also fire on an incidental
@@ -225,31 +358,48 @@ export class ResearchSynthesisService implements IService {
       if (citedHeaderIdx >= 0) {
         logger.debug('[ResearchSynthesisService] Replacing existing CITED LINKS with verified version');
         // The verified list is renumbered (dedup by URL, implausible entries dropped),
-        // but the body's inline [N] markers were authored against the list being
-        // replaced. Remap them by URL identity — parse the synthesis's own list and
-        // map each written number to the global id of the same normalized URL. In
-        // deep mode the numbering already matches and this is a no-op; in quick mode
-        // (where the model's own list is what gets replaced) it prevents every
-        // marker after a dropped/deduped entry from pointing at the wrong source.
+        // but the body's inline [N] markers were authored against the list being replaced.
+        // Remap them by URL identity — parse the synthesis's own list and map each
+        // written number to the global id of the same normalized URL. In deep mode
+        // the numbering already matches and this is a no-op; in quick mode (where the
+        // model's own list is what gets replaced) it prevents every marker after a
+        // dropped/deduped entry from pointing at the wrong source.
         const ownCitations = parseCitations(synthesis);
         const urlToGlobalId = new Map(globalCitations.map((c) => [normalizeUrl(c.url), c.id]));
         const localToGlobal = new Map<number, number>();
+        // own-citation numbers whose URL is absent from the verified list. Their inline
+        // [N] markers are REMOVED cleanly (never '[removed]') so the prose reads
+        // naturally. A marker can dangle only if the lead cited a URL absent from the
+        // rebuilt list (e.g. it invented one not in any researcher report); removing it
+        // beats shipping a broken token or silently retargeting it at the wrong source.
+        const droppedNumbers = new Set<number>();
         ownCitations.forEach((cit, index) => {
+          const localKey = cit.number ?? index + 1;
           const globalId = urlToGlobalId.get(normalizeUrl(cit.url));
-          if (globalId !== undefined) localToGlobal.set(cit.number ?? index + 1, globalId);
+          if (globalId !== undefined) localToGlobal.set(localKey, globalId);
+          else droppedNumbers.add(localKey);
         });
         let body = synthesis.slice(0, citedHeaderIdx).trimEnd();
-        const needsRemap = [...localToGlobal].some(([local, global]) => local !== global);
-        if (needsRemap) {
-          body = body.replace(/\[(\d+)\]/g, (match, p1) => {
-            const globalId = localToGlobal.get(parseInt(p1, 10));
-            return globalId !== undefined ? `[${globalId}]` : match;
-          });
+        if (localToGlobal.size > 0 || droppedNumbers.size > 0) {
+          body = rewriteCitationMarkers(body, localToGlobal, (n) => droppedNumbers.has(n));
         }
+        // Defense-in-depth: redact unverified inline URLs in the prose body. The body
+        // is otherwise passed through verbatim, so this is the one channel by which a
+        // fabricated source can still reach the user; inline URLs also violate the
+        // required format. Verified inline references (in `trustedUrls`) are kept.
+        body = redactUnverifiedProseUrls(body, trustedUrls);
         return `${body}\n\n${verifiedLinksSection}`;
       }
       logger.warn('[ResearchSynthesisService] Synthesis missing CITED LINKS - appending verified version');
-      return `${synthesis.trim()}\n\n${verifiedLinksSection}`;
+      // No existing CITED LINKS header: the synthesis is all prose. The model's
+      // inline [N] markers were authored without a list we could remap against, so
+      // REMOVE any that fall outside the verified range (1..K) — they point at sources
+      // absent from the rebuilt list — before sweeping for unverified inline URLs and
+      // appending the verified section. Removed cleanly (never '[removed]').
+      const validIds = new Set(globalCitations.map((c) => c.id));
+      let prose = rewriteCitationMarkers(synthesis.trim(), new Map(), (n) => !validIds.has(n));
+      prose = redactUnverifiedProseUrls(prose, trustedUrls);
+      return `${prose}\n\n${verifiedLinksSection}`;
     }
 
     // Genuinely no sources were retrieved (e.g. every search/scrape was blocked by
@@ -257,7 +407,13 @@ export class ResearchSynthesisService implements IService {
     // uncited report that looks sourced — but also never destroy a CITED LINKS block
     // the model may have written that we simply could not parse (preserve as-is).
     if (/CITED LINKS/i.test(synthesis)) {
-      return synthesis;
+      // Preserved, but NOT silently: reaching here means no citation parsed AND no
+      // scrape succeeded, so every link in that block is unverified by construction.
+      // Returning it untouched shipped a fully sourced-looking report with no caveat —
+      // the worst case for a reader, since an unretrievable session is precisely when
+      // a model's links are most likely to be invented.
+      logger.warn('[ResearchSynthesisService] Preserving an unparseable CITED LINKS block from a run that retrieved no sources — flagging it as unverified.');
+      return `${synthesis.trimEnd()}\n\n${UNVERIFIED_LINKS_NOTE}`;
     }
     logger.warn('[ResearchSynthesisService] No citations and no scrape provenance — appending explicit no-sources note.');
     return `${synthesis.trim()}\n\n${NO_SOURCES_NOTE}`;

package/src/web-research/retry-utils.ts

@@ -39,9 +39,15 @@ export interface RetryOptions {
 
 /**
  * Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal`
- * aborts. The timer is unref'd so it never keeps the event loop alive.
+ * aborts. The timer is unref'd by default so it never keeps the event loop alive.
+ *
+ * Pass `keepAlive` when the sleep is *foreground* work — i.e. the caller is
+ * mid-operation and the operation must still complete after the delay. An unref'd
+ * timer is only safe while some other referenced handle exists; if the delay is the
+ * sole pending handle, Node drains the loop and exits mid-operation, silently
+ * truncating the work instead of resuming it.
  */
-export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
+export function abortableDelay(ms: number, signal?: AbortSignal, keepAlive = false): Promise<void> {
   return new Promise<void>((resolve, reject) => {
     if (signal?.aborted) {
       reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }));
@@ -51,7 +57,7 @@ export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void>
       if (signal && onAbort) signal.removeEventListener('abort', onAbort);
       resolve();
     }, ms);
-    (timeoutId as TimeoutHandle).unref?.();
+    if (!keepAlive) (timeoutId as TimeoutHandle).unref?.();
     const onAbort = signal
       ? () => {
           clearTimeout(timeoutId as unknown as ReturnType<typeof setTimeout>);

package/src/web-research/scraper-utils.ts

@@ -264,6 +264,49 @@ export async function getSsrfSafeDispatcher(): Promise<unknown | null> {
   return cachedSsrfDispatcher as unknown | null;
 }
 
+/**
+ * Render an error for diagnostics, surfacing the swallowed underlying cause.
+ *
+ * Node's `fetch()`/undici wraps the real transport reason (ECONNRESET, ENOTFOUND,
+ * a TLS error, UND_ERR_*, an AbortError) under `error.cause`, and the top-level
+ * object is almost always the uninformative `TypeError: fetch failed`. Logging
+ * only `String(err)` (or `err.message`) therefore records "fetch failed" and
+ * discards the actual reason — which made the production scrape-failure spike
+ * (212+ sub-5ms failures) impossible to diagnose from logs. This walks the
+ * `cause` chain (bounded, mirroring `isTransientError`) and folds in any Node /
+ * undici `.code`, so each "fetch failed" log line is self-diagnosing.
+ *
+ * Defensive against throwing getters and non-Error values: never throws.
+ */
+export function formatErrorWithCause(err: unknown): string {
+  try {
+    const parts: string[] = [];
+    let current: unknown = err;
+    for (let depth = 0; current !== null && current !== undefined && depth < 5; depth++) {
+      let segment: string;
+      if (current instanceof Error) {
+        segment = current.message || current.name;
+        const code = (current as { code?: unknown }).code;
+        if (typeof code === 'string' && code && !segment.includes(code)) {
+          segment = `${segment} [${code}]`;
+        }
+      } else if (typeof current === 'string') {
+        segment = current;
+      } else {
+        segment = String(current);
+      }
+      if (segment && !parts.includes(segment)) parts.push(segment);
+      const next = (current as { cause?: unknown }).cause;
+      if (next === current) break; // defensive: self-referential cause
+      current = next;
+    }
+    if (parts.length === 0) return String(err);
+    return parts.length === 1 ? parts[0]! : parts.join(' ← ');
+  } catch {
+    return String(err);
+  }
+}
+
 /**
  * Check if an IPv4 address is private/reserved/loopback.
  */

package/src/web-research/web-scraper.ts

@@ -23,12 +23,16 @@ import {
 import {
   FETCH_LAYER_TIMEOUT,
 } from './types.ts';
-import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, isBenignScrapeFailure, } from './scraper-utils.ts';
+import { getRandomUserAgent, extractDomain, validateUrlForSSRF, validateContent, createNativeMarkdownConverter, createJsMarkdownConverter, getSsrfSafeDispatcher, formatErrorWithCause, isBenignScrapeFailure, } from './scraper-utils.ts';
+import { isTransientError, abortableDelay } from './retry-utils.ts';
 import { safeUnref } from '../utils/safe-unref.ts';
 import { getServiceContainer } from '../core/service-registry.ts';
 import type { ServiceContainer } from '../core/service-registry.ts';
 import { cacheScrapedContent } from '../utils/shared-links.ts';
 
+/** Backoff before the fetch layer's single transient retry. */
+const FETCH_RETRY_DELAY_MS = 250;
+
 let playwrightAvailable: boolean = false;
 let markdownConverterPromise: Promise<(html: string) => Promise<string>> | null = null;
 
@@ -191,6 +195,25 @@ async function readBodyCapped(response: Response, maxBytes: number): Promise<Uin
   return out;
 }
 
+/**
+ * Record a fetch-layer failure exactly once per URL.
+ *
+ * Kept out of {@link scrapeWithFetch} because that function is retried: tracking
+ * inside it would file one transient failure as two, and the fetch layer's error
+ * counts are precisely what production incidents are diagnosed from.
+ */
+function trackFetchLayerFailure(url: string, error: unknown): void {
+  const isSsrf = error instanceof Error && error.message.includes('not allowed');
+  errorTracker.trackError(error instanceof Error ? error : String(error), {
+    component: 'scrapers',
+    operation: 'fetch',
+    url,
+    domain: extractDomain(url),
+    layer: 'fetch',
+    ...(isSsrf ? { errorType: 'ssrf_blocked' } : {}),
+  });
+}
+
 async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<ScrapeLayerResult> {
   await validateUrlForSSRF(url);
 
@@ -313,14 +336,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
         const sizeMB = Math.round(e.bytesRead / 1024 / 1024);
         logger.warn(`[Scrapers] HTML response too large (streamed ${sizeMB}MB, max 25MB), skipping`);
         metrics.increment('scrape_errors_total', 1, { error_type: 'size_exceeded', content_type: 'html' });
-        errorTracker.trackError(new Error(`HTML response too large (${sizeMB}MB, max 25MB)`), {
-          component: 'scrapers',
-          operation: 'fetch',
-          url,
-          domain: extractDomain(url),
-          layer: 'fetch',
-          contentType: 'html',
-        });
+        // Not tracked here — scrapeSingle tracks the fetch layer's final error once.
         throw new Error(`HTML response too large (${sizeMB}MB, max 25MB)`, { cause: e });
       }
       throw e;
@@ -330,14 +346,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
       const sizeMB = Math.round(bodyBytes.length / 1024 / 1024);
       logger.warn(`[Scrapers] HTML response too large (actual: ${sizeMB}MB, max 25MB), skipping`);
       metrics.increment('scrape_errors_total', 1, { error_type: 'size_exceeded', content_type: 'html' });
-      errorTracker.trackError(new Error(`HTML response too large (${sizeMB}MB, max 25MB)`), {
-        component: 'scrapers',
-        operation: 'fetch',
-        url,
-        domain: extractDomain(url),
-        layer: 'fetch',
-        contentType: 'html',
-      });
+      // Not tracked here — scrapeSingle tracks the fetch layer's final error once.
       throw new Error(`HTML response too large (${sizeMB}MB, max 25MB)`);
     }
 
@@ -356,27 +365,17 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
     metrics.observe('scrape_latency_ms', fetchDuration, { layer: 'fetch', content_type: 'html', status: 'success' });
     return { source: 'fetch', layer: 'fetch', markdown };
   } catch (error) {
+    // Attempt-scoped metrics stay here (they count fetch operations, and a retried
+    // attempt IS a second operation). Error TRACKING does not: scrapeSingle wraps this
+    // call in a retry, and tracking per attempt would record one failure twice —
+    // inflating exactly the error counts these numbers are read to diagnose.
+    // scrapeSingle calls trackFetchLayerFailure once with the final error.
     if (error instanceof Error && error.message.includes('not allowed')) {
       metrics.increment('scrape_operations_total', 1, { layer: 'fetch', status: 'ssrf_blocked' });
       metrics.observe('scrape_latency_ms', Date.now() - fetchStart, { layer: 'fetch', status: 'ssrf_blocked' });
-      errorTracker.trackError(error, {
-        component: 'scrapers',
-        operation: 'fetch',
-        url,
-        domain: extractDomain(url),
-        layer: 'fetch',
-        errorType: 'ssrf_blocked',
-      });
       throw error;
     }
     metrics.increment('scrape_operations_total', 1, { layer: 'fetch', status: 'error' });
-    errorTracker.trackError(error instanceof Error ? error : String(error), {
-      component: 'scrapers',
-      operation: 'fetch',
-      url,
-      domain: extractDomain(url),
-      layer: 'fetch',
-    });
     throw error;
   } finally {
     clearTimeout(timeoutId);
@@ -482,7 +481,35 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
   
   const start = Date.now();
   try {
-    const res = await scrapeWithFetch(url, signal);
+    // One cheap transient retry before falling back to the (10–30× slower) stealth
+    // browser. The fetch layer is the only scrape layer with no retry; a transient
+    // socket/DNS blip (ECONNRESET, ENOTFOUND, UND_ERR_*, a bare `fetch failed`)
+    // otherwise costs a full browser render. Gated on isTransientError, so SSRF
+    // rejections, size caps and 4xx rethrow immediately; the outer catch still owns
+    // the browser fallback on genuine exhaustion.
+    //
+    // Deliberately NOT retryWithBackoff: it logs every retry at WARN, and a
+    // fetch-layer miss is an expected, benign event with a fallback behind it —
+    // the same reason isBenignScrapeFailure keeps these out of ERROR. A burst of
+    // blocked scrapes would otherwise emit one WARN per URL.
+    const res = await (async () => {
+      try {
+        return await scrapeWithFetch(url, signal);
+      } catch (firstErr) {
+        if (!isTransientError(firstErr) || signal?.aborted) throw firstErr;
+        logger.debug(`[Scrapers] fetch transient failure for ${url}, retrying once: ${formatErrorWithCause(firstErr)}`);
+        metrics.increment('scrape_fetch_retries_total', 1, { layer: 'fetch' });
+        try {
+          // keepAlive: the retry is foreground work. An unref'd backoff is only safe
+          // while some other handle is referenced; a lone scrape would otherwise let
+          // the process exit during the 250ms gap and never make the second attempt.
+          await abortableDelay(FETCH_RETRY_DELAY_MS, signal, true);
+        } catch {
+          throw firstErr; // aborted during backoff — surface the real cause
+        }
+        return await scrapeWithFetch(url, signal);
+      }
+    })();
     const duration = Date.now() - start;
     logger.log(`[Scrapers] fetch success for ${url} in ${duration}ms`);
     metrics.increment('scrape_results_total', 1, { outcome: 'fetch_success' });
@@ -494,10 +521,10 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
     return { ...res, url, success: true };
   } catch (e1) {
     const fetchDuration = Date.now() - start;
-    logger.debug(`[Scrapers] fetch failed for ${url} in ${fetchDuration}ms: ${String(e1)}`);
-    // Not re-tracked here: scrapeWithFetch's own catch already recorded this error
-    // with richer layer/errorType context immediately before rethrowing. Tracking
-    // it again at this frame would double-count the same failure.
+    logger.debug(`[Scrapers] fetch failed for ${url} in ${fetchDuration}ms: ${formatErrorWithCause(e1)}`);
+    // Tracked here rather than inside scrapeWithFetch, which is retried above —
+    // one transient failure must file one error, not one per attempt.
+    trackFetchLayerFailure(url, e1);
 
     if (playwrightAvailable) {
       try {
@@ -524,18 +551,18 @@ export async function scrapeSingle(url: string, signal?: AbortSignal, config?: C
         // at ERROR with its stack. Either way, scrapeWithStealthBrowser's own catch
         // already tracked the error with layer context, so it is not re-tracked here.
         if (isBenignScrapeFailure(e2)) {
-          logger.debug(`[Scrapers] Browser fallback did not yield content for ${url} in ${totalDuration}ms: ${String(e2)}`);
+          logger.debug(`[Scrapers] Browser fallback did not yield content for ${url} in ${totalDuration}ms: ${formatErrorWithCause(e2)}`);
         } else {
           logger.error(`[Scrapers] Browser fallback failed for ${url} in ${totalDuration}ms:`, e2);
         }
         metrics.increment('scrape_errors_total', 1, { error_type: 'fallback_failed', layer: 'playwright' });
         metrics.increment('scrape_results_total', 1, { outcome: 'total_failure' });
-        return { url, success: false, error: String(e2), markdown: '' };
+        return { url, success: false, error: formatErrorWithCause(e2), markdown: '' };
       }
     }
     metrics.increment('scrape_errors_total', 1, { error_type: 'no_fallback_available', layer: 'fetch' });
     metrics.increment('scrape_results_total', 1, { outcome: 'total_failure' });
-    return { url, success: false, error: String(e1), markdown: '' };
+    return { url, success: false, error: formatErrorWithCause(e1), markdown: '' };
   }
 }