@lincoln504/pi-research 1.0.12 → 1.1.2

package/src/infrastructure/browser/task-execution-service.ts

@@ -22,6 +22,46 @@ import type { ISchedulerFactory, IScheduler } from '../../core/scheduler-factory
 let lastRestartTime = 0;
 const RESTART_COOLDOWN_MS = 10000;
 
+/**
+ * Extra attempts reserved for a *leader handover* — the elected browser-pool leader
+ * exiting while this process still has work in flight ("Worker pool is shutting
+ * down" / "draining").
+ *
+ * This has its own budget because it is not a random socket blip: it is a known,
+ * self-healing window that lasts as long as a fresh election takes (measured at
+ * ~4 s on a loaded machine), and it hits *every* in-flight task of the affected
+ * run at once. With only the single generic retry, a whole 20-query search burst
+ * could be wiped out by one leader exiting — the run then failed outright with
+ * "all N queries encountered worker errors", seconds before the new leader came up.
+ * Spending these attempts is strictly cheaper than losing the run.
+ */
+const LEADER_HANDOVER_RETRIES = 3;
+/** Backoff between handover attempts — long enough for an election to complete. */
+const HANDOVER_BACKOFF_MS = 1200;
+
+/**
+ * Recover from a leader handover before retrying.
+ *
+ * Critically this DROPS the cached scheduler handle. A pool-shutdown error means
+ * the leader we are connected to is going away, so the cached client points at a
+ * corpse; simply waiting and retrying reconnects to the same dead port (observed
+ * as a retry logging "Connecting to existing scheduler" with the *same* version id
+ * and failing identically). Clearing it forces the next `getScheduler()` to
+ * re-resolve — joining the new leader, or winning the election itself.
+ *
+ * `forceSchedulerRestart(false)` keeps its own liveness probe and in-progress
+ * guard, so concurrent callers coalesce instead of stampeding.
+ */
+async function recoverFromLeaderHandover(container: ServiceContainer): Promise<void> {
+    logger.warn('[BrowserManager] Leader handover detected (pool draining) — dropping stale scheduler handle and re-resolving...');
+    await forceSchedulerRestart(false, container).catch((err) =>
+        logger.debug('[BrowserManager] Scheduler re-resolve during handover failed (non-fatal):', err),
+    );
+    await waitForBrowserPoolIdle(15000).catch((err) =>
+        logger.debug('Wait for browser idle timed out or failed:', err),
+    );
+}
+
 /**
  * Get the scheduler from the factory, optionally using a specific container.
  */
@@ -56,7 +96,8 @@ export async function runBrowserTask<T>(
     config?: Config,
     signal?: AbortSignal,
     retries = 1,
-    container: ServiceContainer = getServiceContainer()
+    container: ServiceContainer = getServiceContainer(),
+    handoverRetries = LEADER_HANDOVER_RETRIES,
 ): Promise<T> {
     if (signal?.aborted) throw new Error('Aborted');
 
@@ -84,6 +125,23 @@ export async function runBrowserTask<T>(
     } catch (error: unknown) {
         if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw new Error('Aborted', { cause: error });
 
+        // A leader handover gets its own budget: it is a recoverable, seconds-long
+        // window that strikes every in-flight task at once, so it must not be
+        // charged against (or limited by) the single generic transient retry.
+        const isHandover = isPoolShutdownError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error);
+        if (isHandover && handoverRetries > 0) {
+            errorTracker.trackError(error, {
+                component: 'browser-manager',
+                operation: type,
+                taskType: type,
+                errorType: 'leader_handover',
+            });
+            logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
+            await recoverFromLeaderHandover(container);
+            await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
+            return runBrowserTask<T>(taskOrUrl, type, config, signal, retries, container, handoverRetries - 1);
+        }
+
         if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
             errorTracker.trackError(error, {
                 component: 'browser-manager',
@@ -92,9 +150,9 @@ export async function runBrowserTask<T>(
                 errorType: 'transient_socket_error',
             });
             logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
-            
+
             if (isPoolShutdownError(error)) {
-                // Pool is temporarily draining — wait for the drain to finish.
+                // Handover budget already spent — fall back to a plain drain wait.
                 logger.warn(`[BrowserManager] Pool is draining — waiting for pool idle before retry...`);
                 await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
             } else {
@@ -120,7 +178,7 @@ export async function runBrowserTask<T>(
             // Retry with jitter (100-500ms) to prevent thundering herd
             const jitter = 100 + Math.floor(Math.random() * 400);
             await new Promise(resolve => setTimeout(resolve, jitter));
-            return runBrowserTask<T>(taskOrUrl, type, config, signal, retries - 1, container);
+            return runBrowserTask<T>(taskOrUrl, type, config, signal, retries - 1, container, handoverRetries);
         }
         throw error;
     }
@@ -143,9 +201,12 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
                 errorType: 'transient_socket_error',
             });
             logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
-            
+
             if (isPoolShutdownError(error)) {
-                await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
+                // Same stale-handle trap as the task paths: waiting alone leaves the
+                // cached client aimed at the departing leader, so the retry re-probes
+                // a corpse and reports the pool unhealthy during a routine handover.
+                await recoverFromLeaderHandover(container);
             } else {
                 // See runBrowserTask: don't force-clear remote leader state on a bare
                 // transient socket error — let the liveness probe inside
@@ -172,7 +233,7 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
 /**
  * Run a worker search query with retry logic.
  */
-export async function runWorkerSearch(query: string, config?: Config, signal?: AbortSignal, retries = 1, sessionId?: string, container: ServiceContainer = getServiceContainer()): Promise<SearchResult[]> {
+export async function runWorkerSearch(query: string, config?: Config, signal?: AbortSignal, retries = 1, sessionId?: string, container: ServiceContainer = getServiceContainer(), handoverRetries = LEADER_HANDOVER_RETRIES): Promise<SearchResult[]> {
     if (signal?.aborted) throw new Error('Aborted');
     
     const breaker = sessionId ? getBrowserCircuitBreaker(sessionId) : browserCircuitBreaker;
@@ -186,6 +247,22 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
     } catch (error: unknown) {
         if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw new Error('Aborted', { cause: error });
 
+        // See runBrowserTask: a leader handover wipes out an entire search burst at
+        // once, so it gets its own attempts rather than the single generic retry.
+        const isHandover = isPoolShutdownError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error);
+        if (isHandover && handoverRetries > 0) {
+            errorTracker.trackError(error, {
+                component: 'browser-manager',
+                operation: 'search',
+                query,
+                errorType: 'leader_handover',
+            });
+            logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
+            await recoverFromLeaderHandover(container);
+            await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
+            return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
+        }
+
         if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
             errorTracker.trackError(error, {
                 component: 'browser-manager',
@@ -194,7 +271,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
                 errorType: 'transient_socket_error',
             });
             logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
-            
+
             if (isPoolShutdownError(error)) {
                 await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
             } else {
@@ -214,7 +291,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
 
             const jitter = 100 + Math.floor(Math.random() * 400);
             await new Promise(resolve => setTimeout(resolve, jitter));
-            return runWorkerSearch(query, config, signal, retries - 1, sessionId, container);
+            return runWorkerSearch(query, config, signal, retries - 1, sessionId, container, handoverRetries);
         }
         throw error;
     }

package/src/infrastructure/browser/thread-worker.mjs

@@ -1284,6 +1284,11 @@ var init_circuit_breaker = __esm({
           metrics.observe("circuit_breaker_call_duration_ms", duration, { breaker: this.options.name, status: "error" });
           if (this.options.isTransientError(error)) {
             this.onFailure(error);
+          } else if (admittedAsProbe && this.state === "HALF_OPEN") {
+            logger.warn(
+              `[CircuitBreaker] '${this.options.name}' HALF_OPEN probe failed with a non-counted error; treating as inconclusive and re-opening: ${error instanceof Error ? error.message : String(error)}`
+            );
+            this.transitionTo("OPEN");
           }
           throw error;
         } finally {
@@ -1379,7 +1384,12 @@ function isServerDrainingError(error) {
 function isPoolShutdownError(error) {
   if (!error || typeof error !== "object") return false;
   const err = error;
-  return typeof err.message === "string" && (err.message.includes("Worker pool is shutting down") || err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool"));
+  return typeof err.message === "string" && (err.message.includes("Worker pool is shutting down") || err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // worker-pool-manager throws this when auto-recovery is still swapping the
+  // pool. Its own text says "please retry", but until it was listed here no
+  // classifier matched it, so the first attempt rethrew instead of retrying —
+  // and auto-recovery fires only after 3 consecutive worker failures, i.e.
+  // exactly when a burst of tasks is in flight and would all die together.
+  err.message.includes("Worker pool is being reset"));
 }
 var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({

package/src/infrastructure/research-run-semaphore.ts

@@ -21,9 +21,10 @@
  *   configured as a *try-take* (`lockTimeout: 0`): it either wins the slot (free,
  *   or reclaimed from a *dead* owner) immediately or throws as soon as it sees a
  *   *live* owner. If every slot is held by a live process, the loop waits
- *   `pollDelayMs` and retries until `maxWaitMs` elapses, then rejects with
- *   {@link ResearchRunCapacityError} (fail-fast with a clear message rather than
- *   a 60-error degraded run).
+ *   `pollDelayMs` and retries until `maxWaitMs` elapses (by default
+ *   {@link DEFAULT_ACQUIRE_TIMEOUT_MS} — extra runs **queue** behind the in-flight
+ *   ones instead of failing), then rejects with {@link ResearchRunCapacityError}
+ *   rather than degrading into a 60-error run.
  * - **Release** unlinks the slot file via the captured slot service.
  * - **Crash recovery** is automatic and PID-reuse-safe: a slot held by a process
  *   that has died (killed, crashed, SIGKILLed by earlyoom) is reclaimed
@@ -69,9 +70,21 @@ import type { IService } from '../core/service-registry.ts';
 import type { IProcessLifecycle } from '../core/interfaces/process-interfaces.ts';
 
 /** Default maximum number of concurrent research runs across all processes. */
-export const DEFAULT_MAX_CONCURRENT_RUNS = 2;
-/** Default per-acquire wait before failing fast when all slots are busy (0 = fail-fast). */
-export const DEFAULT_ACQUIRE_TIMEOUT_MS = 0;
+export const DEFAULT_MAX_CONCURRENT_RUNS = 3;
+/**
+ * Default per-acquire wait before giving up when all slots are held by live runs.
+ *
+ * This is deliberately a *queue*, not a fail-fast. The cap exists to stop N runs
+ * saturating the shared browser/embedding pool — but an extra run is a legitimate
+ * request to do work, not an error, so the correct response to contention is to
+ * serialize it behind the in-flight runs rather than to fail it. With a depth-1 run
+ * measured at ~2-3 minutes, 10 minutes absorbs a backlog of roughly three further
+ * runs per slot while still resolving well inside the agent-skill's documented
+ * command timeouts (so a queued run never looks like a hang to the caller).
+ *
+ * Set `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS=0` to restore strict fail-fast.
+ */
+export const DEFAULT_ACQUIRE_TIMEOUT_MS = 600_000;
 /** Poll interval between full slot sweeps while waiting for a slot. */
 const POLL_DELAY_MS = 250;
 
@@ -84,9 +97,10 @@ export class ResearchRunCapacityError extends Error {
   readonly slots: number;
   constructor(slots: number, maxWaitMs: number) {
     super(
-      `Maximum concurrent research runs (${slots}) reached${maxWaitMs > 0 ? ` after waiting ${maxWaitMs}ms` : ''}; ` +
-        `wait for an in-progress run to finish and retry. ` +
-        `(Override with PI_RESEARCH_MAX_CONCURRENT_RUNS.)`,
+      `Maximum concurrent research runs (${slots}) reached${maxWaitMs > 0 ? ` and still busy after queueing for ${Math.round(maxWaitMs / 1000)}s` : ''}; ` +
+        `another run is still in progress. This is temporary — retry shortly. ` +
+        `(Raise the cap with PI_RESEARCH_MAX_CONCURRENT_RUNS, or the queue wait with ` +
+        `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS.)`,
     );
     this.name = 'ResearchRunCapacityError';
     this.slots = slots;
@@ -209,10 +223,17 @@ export class ResearchRunSemaphore implements IService {
    * @param signal aborts the wait. Without it a run started under a long
    *   `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS` would ignore Ctrl-C until the deadline
    *   passed, because the wait happens before any orchestrator exists to cancel.
+   * @param onQueued invoked at most once, when the first full sweep finds every
+   *   slot busy and the call is about to wait. Lets a front-end tell the user the
+   *   run is queued rather than hung. Never allowed to break the acquire.
    * @throws {ResearchRunCapacityError} if no slot frees within `maxWaitMs`.
    * @throws {Error} named `AbortError` if `signal` aborts while waiting.
    */
-  async acquire(maxWaitMs: number = this.defaultMaxWaitMs, signal?: AbortSignal): Promise<RunSlotAcquisition> {
+  async acquire(
+    maxWaitMs: number = this.defaultMaxWaitMs,
+    signal?: AbortSignal,
+    onQueued?: (slots: number, maxWaitMs: number) => void,
+  ): Promise<RunSlotAcquisition> {
     if (!this._initialized) await this.initialize();
     if (signal?.aborted) {
       metrics.increment('research_run_acquire_total', 1, { status: 'aborted' });
@@ -220,6 +241,7 @@ export class ResearchRunSemaphore implements IService {
     }
 
     const deadline = Date.now() + Math.max(0, maxWaitMs);
+    let announcedQueued = false;
     for (;;) {
       // Round-robin from a rotating start so contending processes don't synchronize on slot 0.
       const start = this.nextStartIndex;
@@ -238,6 +260,17 @@ export class ResearchRunSemaphore implements IService {
         metrics.increment('research_run_acquire_total', 1, { status: 'exhausted' });
         throw new ResearchRunCapacityError(this.maxSlots, maxWaitMs);
       }
+      // Announce the queue exactly once, on the first sweep that found nothing free.
+      // A throwing front-end callback must never fail the acquire.
+      if (!announcedQueued) {
+        announcedQueued = true;
+        logger.info(`[ResearchRunSemaphore] All ${this.maxSlots} run slot(s) busy — queueing for up to ${maxWaitMs}ms.`);
+        try {
+          onQueued?.(this.maxSlots, maxWaitMs);
+        } catch (err) {
+          logger.debug(`[ResearchRunSemaphore] onQueued callback threw: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      }
       // Count the aborted outcome too, so acquired + exhausted + aborted accounts for
       // every acquire attempt. A cancel that vanished from the counter would make the
       // capacity metrics quietly under-report demand.

package/src/orchestration/headless-observer.ts

@@ -95,6 +95,10 @@ export class HeadlessObserver implements ResearchObserver {
     this.emit('start', { query, complexity });
   }
 
+  onRunQueued(slots: number, maxWaitMs: number): void {
+    this.emit('run_queued', { slots, maxWaitMs });
+  }
+
   onPlanningStart(attempt: number): void {
     this.emit('planning_start', { attempt });
   }

package/src/orchestration/research-orchestration-service.ts

@@ -108,17 +108,27 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
 
     const researchStart = Date.now();
 
-    // Cross-process run-cap (Phase 1): acquire one of N slots so concurrent
-    // research runs can't saturate the shared leader-elected browser/embedding
-    // pool and degrade into ~60-error failures. Fail-OPEN on internal/IO errors
-    // (a semaphore bug must never break research); only a genuine
-    // ResearchRunCapacityError (all slots held by live processes) propagates,
-    // failing the run fast with a clear message. The slot is released in finally.
+    // Cross-process run-cap: acquire one of N slots so concurrent research runs
+    // can't saturate the shared leader-elected browser/embedding pool and degrade
+    // into ~60-error failures. Over-cap runs QUEUE for a slot (announced once via
+    // the observer) rather than failing, because an extra run is a legitimate
+    // request to do work, not an error. Fail-OPEN on internal/IO errors (a
+    // semaphore bug must never break research); only a genuine
+    // ResearchRunCapacityError — nothing freed within the whole queue window —
+    // propagates. The slot is released in finally.
     let runSlot: { slotIndex: number; release(): Promise<void> } | null = null;
     try {
       const container = tryGetServiceContainerFromCtx(ctx);
       const semaphore = await getService<ResearchRunSemaphore>(ServiceNames.RESEARCH_RUN_SEMAPHORE, ctx, container);
-      runSlot = await semaphore.acquire(undefined, signal);
+      runSlot = await semaphore.acquire(undefined, signal, (slots, maxWaitMs) => {
+        // The queue notice fires BEFORE any orchestrator exists, so `observer` is
+        // still exactly what the caller passed: either a ResearchObserver, or the
+        // HeadlessObserverOptions bag (only `onProgress`) that the CLI supplies and
+        // the orchestrator would normally wrap later. Dispatch to whichever shape
+        // is present — handling only the former silently drops the notice on the
+        // CLI, which is the very front-end that most needs it.
+        notifyRunQueued(observer, slots, maxWaitMs);
+      });
       logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
     } catch (err) {
       if (err instanceof ResearchRunCapacityError) throw err; // capacity exhausted → fail fast
@@ -583,3 +593,28 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
     }
   }
 }
+
+/**
+ * Deliver the "your run is queued behind other runs" notice to whichever observer
+ * shape the caller supplied.
+ *
+ * This fires before the orchestrator normalizes the observer, so both forms are
+ * live here: a {@link ResearchObserver} instance (SDK / pi extension) or the
+ * {@link HeadlessObserverOptions} bag carrying only `onProgress` (the CLI and
+ * agent skill). A queued run is otherwise completely silent for the whole acquire
+ * wait, which is indistinguishable from a hang.
+ *
+ * Never throws: a front-end callback must not be able to fail the run it is
+ * merely reporting on.
+ */
+function notifyRunQueued(observer: unknown, slots: number, maxWaitMs: number): void {
+  try {
+    const obs = observer as
+      | { onRunQueued?: (s: number, w: number) => void; onProgress?: (e: string, d?: unknown) => void }
+      | undefined;
+    if (typeof obs?.onRunQueued === 'function') obs.onRunQueued(slots, maxWaitMs);
+    else if (typeof obs?.onProgress === 'function') obs.onProgress('run_queued', { slots, maxWaitMs });
+  } catch (err) {
+    logger.debug(`[ResearchOrchestrationService] run_queued notification threw: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}

package/src/orchestration/research-synthesis-service.ts

@@ -10,7 +10,7 @@
  */
 
 import { parseCitations } from '../utils/text-utils.ts';
-import { normalizeCitations, formatCitedLinks, type GlobalCitation } from '../utils/citation-utils.ts';
+import { normalizeCitations, formatCitedLinks, rewriteCitationMarkers, type GlobalCitation } from '../utils/citation-utils.ts';
 import { lastCitedLinksHeaderIndex } from '../utils/text-utils.ts';
 import { getScrapedLinks, isTranscribedLink, normalizeUrl } from '../utils/shared-links.ts';
 import { stripTrailingLlmPunctuation } from '../utils/url-utils.ts';
@@ -33,86 +33,6 @@ const NO_SOURCES_NOTE =
 const UNVERIFIED_LINKS_NOTE =
   '_None of the links above could be verified: no page fetch succeeded this session and no citation could be matched to retrieved content. They may be inaccurate or invented — check each one before relying on it._';
 
-/**
- * Largest `[N]` treated as a citation marker. Inline markers index a source list
- * that is realistically well under this; the bound exists so an ordinary bracketed
- * number in prose — a year (`[2023]`), an issue id, an array index — is left alone
- * instead of being silently deleted as a "dangling citation".
- */
-const MAX_PLAUSIBLE_CITATION_MARKER = 200;
-
-/**
- * Rewrite inline `[N]` citation markers in a prose body: renumber the ones we can
- * map, delete the ones known to dangle, leave everything else untouched.
- *
- * Deletion is whitespace-aware rather than followed by a global space collapse.
- * A blanket `text.replace(/  +/g, ' ')` also eats *leading* indentation, which
- * silently flattens nested list levels into one and turns 4-space indented code
- * blocks into paragraphs — corrupting the structure of every report that contains
- * them. So the pad preceding a deleted marker is consumed only when the marker is
- * mid-line (i.e. real prose spacing); line-leading indentation is preserved.
- *
- * Code is skipped entirely. A report may quote code containing bracketed integers
- * (`items[12]`), and on the no-CITED-LINKS path *every* out-of-range marker is treated
- * as dangling — so without this guard a quoted array index would be silently deleted
- * from the user's code sample. Fenced blocks are tracked across lines; single-backtick
- * spans are detected by an odd backtick count earlier on the same line.
- *
- * @param remap  local marker number → verified global id
- * @param isDangling  whether a marker not present in `remap` should be deleted
- */
-function rewriteCitationMarkers(
-  text: string,
-  remap: Map<number, number>,
-  isDangling: (n: number) => boolean,
-): string {
-  const inFence = fencedCodeMask(text);
-  return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad: string, digits: string, offset: number) => {
-    const n = parseInt(digits, 10);
-    const mapped = remap.get(n);
-    if (mapped !== undefined) return `${pad}[${mapped}]`;
-    // Citation numbering is 1-based; `[0]` is always an index or a literal, never a marker.
-    if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
-    if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
-    // Preserve indentation when nothing but whitespace precedes on this line;
-    // otherwise drop the pad too so no double space is left behind.
-    const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
-    return text.slice(lineStart, offset).trim() === '' ? pad : '';
-  });
-}
-
-/**
- * Build an `offset → is inside a ``` fenced block` predicate for `text`.
- *
- * Computed once per rewrite (a single line scan) rather than per marker, so marker
- * rewriting stays linear in the body length.
- */
-function fencedCodeMask(text: string): (offset: number) => boolean {
-  const ranges: Array<[number, number]> = [];
-  let open = -1;
-  let pos = 0;
-  for (const line of text.split('\n')) {
-    if (/^\s{0,3}(```|~~~)/.test(line)) {
-      if (open < 0) open = pos;
-      else {
-        ranges.push([open, pos + line.length]);
-        open = -1;
-      }
-    }
-    pos += line.length + 1; // +1 for the '\n' consumed by split
-  }
-  // An unterminated fence runs to the end of the body — treat the remainder as code.
-  if (open >= 0) ranges.push([open, text.length]);
-  return (offset: number) => ranges.some(([start, end]) => offset >= start && offset < end);
-}
-
-/** Whether `offset` sits inside a single-backtick code span on its own line. */
-function inInlineCodeSpan(text: string, offset: number): boolean {
-  const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
-  let ticks = 0;
-  for (let i = lineStart; i < offset; i++) if (text[i] === '`') ticks++;
-  return ticks % 2 === 1;
-}
 
 /**
  * Redact inline `https?://` tokens in synthesis prose whose URL was NOT retrieved

package/src/tools/research-knowledge-search.ts

@@ -481,7 +481,10 @@ export async function runBackgroundExtraction(
       const delay = KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS * attempt;
       metrics.increment('research_knowledge_search_synthesis_retries_total', 1);
       logger.warn(`[research-knowledge-search] Synthesis attempt ${attempt}/${KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS} failed transiently (${msg}); retrying in ${delay}ms`);
-      await abortableDelay(delay, signal);
+      // keepAlive=true: the retry is already committed and the next attempt must
+      // run, so this sleep is foreground work. An unref'd timer here can be the
+      // only pending handle and let the process exit mid-retry (see abortableDelay).
+      await abortableDelay(delay, signal, true);
     }
   }
 
@@ -625,7 +628,10 @@ export async function triageRelevantUrls(
       const delay = KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS * attempt;
       metrics.increment('research_knowledge_search_triage_retries_total', 1);
       logger.warn(`[research-knowledge-search] Triage attempt ${attempt}/${KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS} failed transiently (${msg}); retrying in ${delay}ms`);
-      await abortableDelay(delay, signal);
+      // keepAlive=true: the retry is already committed and the next attempt must
+      // run, so this sleep is foreground work. An unref'd timer here can be the
+      // only pending handle and let the process exit mid-retry (see abortableDelay).
+      await abortableDelay(delay, signal, true);
     }
   }
 

package/src/utils/circuit-breaker.ts

@@ -78,6 +78,26 @@ export class CircuitBreaker {
       metrics.observe('circuit_breaker_call_duration_ms', duration, { breaker: this.options.name, status: 'error' });
       if (this.options.isTransientError(error)) {
         this.onFailure(error);
+      } else if (admittedAsProbe && this.state === 'HALF_OPEN') {
+        // The trial probe failed for a reason this breaker does not count. That is
+        // INCONCLUSIVE, not healthy — so re-open and wait out another reset window
+        // rather than staying HALF_OPEN.
+        //
+        // Without this the breaker pins permanently: onFailure is the only path to
+        // OPEN, and it is skipped for non-counted errors, while transitionTo(HALF_OPEN)
+        // never re-arms nextAttemptTime. The admission gate above then fast-fails every
+        // caller beyond halfOpenMaxCalls (default 1) forever. This is not a corner case
+        // for the browser pool: DEFAULT_BREAKER_CONFIG.isTransientError deliberately
+        // excludes pool-shutdown, draining, Cloudflare and task-timeout errors — exactly
+        // what a recovering or blocked pool returns — so the probe most likely to run is
+        // also the one most likely to pin the breaker. Observed effect: 19 of a 20-query
+        // parallel burst fast-failing instantly, surfacing to the researcher as
+        // "your query may be too narrow".
+        logger.warn(
+          `[CircuitBreaker] '${this.options.name}' HALF_OPEN probe failed with a non-counted error; ` +
+            `treating as inconclusive and re-opening: ${error instanceof Error ? error.message : String(error)}`
+        );
+        this.transitionTo('OPEN');
       }
       throw error;
     } finally {

package/src/utils/citation-utils.ts

@@ -67,13 +67,28 @@ export function normalizeCitations(reports: Map<string, string>): {
     const headerIdx = lastCitedLinksHeaderIndex(report);
     let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
 
-    // Replace [N] with [GlobalID]
-    // Also handle [N][M] combinations
-    content = content.replace(/\[(\d+)\]/g, (match, p1) => {
-      const localId = parseInt(p1, 10);
-      const globalId = localToGlobal.get(localId);
-      return globalId !== undefined ? `[${globalId}]` : match;
-    });
+    // Renumber local [N] markers to their global ids, and DELETE the ones that could
+    // not be mapped.
+    //
+    // Leaving an unmapped marker in place (the previous `return match`) was a silent
+    // misattribution bug, not a harmless no-op: global ids are assigned densely from 1,
+    // so a surviving `[1]` no longer means "this report's first source" — it resolves
+    // against the renumbered global list and points at a completely different document.
+    // Markers go unmapped whenever parseCitations dropped the entry (e.g. a URL whose
+    // hostname has no dot: `localhost`, a bare internal host, a malformed line). The
+    // result is fed straight into the evaluator/synthesis prompt, which is explicitly
+    // told the numbering is already normalized — so the model faithfully reproduces the
+    // wrong attribution, and ensureCitedLinks then regenerates a CITED LINKS list that
+    // looks perfectly consistent with it.
+    //
+    // rewriteCitationMarkers is the same routine the synthesis path already uses: it
+    // skips fenced blocks and inline code spans (so a quoted `items[12]` survives),
+    // bounds what counts as a marker, and preserves line-leading indentation.
+    content = rewriteCitationMarkers(
+      content,
+      localToGlobal,
+      (n) => !localToGlobal.has(n),
+    );
 
     normalizedReports.set(id, content.trim());
   }
@@ -95,3 +110,84 @@ export function formatCitedLinks(citations: GlobalCitation[]): string {
 
   return `CITED LINKS\n${links.join('\n')}`;
 }
+
+/**
+ * Largest `[N]` treated as a citation marker. Inline markers index a source list
+ * that is realistically well under this; the bound exists so an ordinary bracketed
+ * number in prose — a year (`[2023]`), an issue id, an array index — is left alone
+ * instead of being silently deleted as a "dangling citation".
+ */
+export const MAX_PLAUSIBLE_CITATION_MARKER = 200;
+
+/**
+ * Rewrite inline `[N]` citation markers in a prose body: renumber the ones we can
+ * map, delete the ones known to dangle, leave everything else untouched.
+ *
+ * Deletion is whitespace-aware rather than followed by a global space collapse.
+ * A blanket `text.replace(/  +/g, ' ')` also eats *leading* indentation, which
+ * silently flattens nested list levels into one and turns 4-space indented code
+ * blocks into paragraphs — corrupting the structure of every report that contains
+ * them. So the pad preceding a deleted marker is consumed only when the marker is
+ * mid-line (i.e. real prose spacing); line-leading indentation is preserved.
+ *
+ * Code is skipped entirely. A report may quote code containing bracketed integers
+ * (`items[12]`), and on the no-CITED-LINKS path *every* out-of-range marker is treated
+ * as dangling — so without this guard a quoted array index would be silently deleted
+ * from the user's code sample. Fenced blocks are tracked across lines; single-backtick
+ * spans are detected by an odd backtick count earlier on the same line.
+ *
+ * @param remap  local marker number → verified global id
+ * @param isDangling  whether a marker not present in `remap` should be deleted
+ */
+export function rewriteCitationMarkers(
+  text: string,
+  remap: Map<number, number>,
+  isDangling: (n: number) => boolean,
+): string {
+  const inFence = fencedCodeMask(text);
+  return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad: string, digits: string, offset: number) => {
+    const n = parseInt(digits, 10);
+    const mapped = remap.get(n);
+    if (mapped !== undefined) return `${pad}[${mapped}]`;
+    // Citation numbering is 1-based; `[0]` is always an index or a literal, never a marker.
+    if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
+    if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
+    // Preserve indentation when nothing but whitespace precedes on this line;
+    // otherwise drop the pad too so no double space is left behind.
+    const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
+    return text.slice(lineStart, offset).trim() === '' ? pad : '';
+  });
+}
+
+/**
+ * Build an `offset → is inside a ``` fenced block` predicate for `text`.
+ *
+ * Computed once per rewrite (a single line scan) rather than per marker, so marker
+ * rewriting stays linear in the body length.
+ */
+function fencedCodeMask(text: string): (offset: number) => boolean {
+  const ranges: Array<[number, number]> = [];
+  let open = -1;
+  let pos = 0;
+  for (const line of text.split('\n')) {
+    if (/^\s{0,3}(```|~~~)/.test(line)) {
+      if (open < 0) open = pos;
+      else {
+        ranges.push([open, pos + line.length]);
+        open = -1;
+      }
+    }
+    pos += line.length + 1; // +1 for the '\n' consumed by split
+  }
+  // An unterminated fence runs to the end of the body — treat the remainder as code.
+  if (open >= 0) ranges.push([open, text.length]);
+  return (offset: number) => ranges.some(([start, end]) => offset >= start && offset < end);
+}
+
+/** Whether `offset` sits inside a single-backtick code span on its own line. */
+function inInlineCodeSpan(text: string, offset: number): boolean {
+  const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
+  let ticks = 0;
+  for (let i = lineStart; i < offset; i++) if (text[i] === '`') ticks++;
+  return ticks % 2 === 1;
+}