@lincoln504/pi-research 1.0.10 → 1.1.0

package/dist/cli.mjs

@@ -1394,7 +1394,9 @@ var init_service_names = __esm({
       WORKER_POOL_MANAGER: "worker-pool-manager",
       // Research session services
       RESEARCH_SESSION_SERVICE: "research-session-service",
-      RESEARCH_SYNTHESIS_SERVICE: "research-synthesis-service"
+      RESEARCH_SYNTHESIS_SERVICE: "research-synthesis-service",
+      // Cross-process cap on concurrent research runs (Phase 1 run-cap)
+      RESEARCH_RUN_SEMAPHORE: "research-run-semaphore"
     };
   }
 });
@@ -3370,7 +3372,7 @@ var init_model_registry_factory = __esm({
 });
 
 // src/web-research/retry-utils.ts
-function abortableDelay(ms, signal) {
+function abortableDelay(ms, signal, keepAlive = false) {
   return new Promise((resolve8, reject) => {
     if (signal?.aborted) {
       reject(Object.assign(new Error("Aborted"), { name: "AbortError" }));
@@ -3380,7 +3382,7 @@ function abortableDelay(ms, signal) {
       if (signal && onAbort) signal.removeEventListener("abort", onAbort);
       resolve8();
     }, ms);
-    timeoutId.unref?.();
+    if (!keepAlive) timeoutId.unref?.();
     const onAbort = signal ? () => {
       clearTimeout(timeoutId);
       reject(Object.assign(new Error("Aborted"), { name: "AbortError" }));
@@ -10668,7 +10670,18 @@ var BrowserClient = class {
           const duration = Date.now() - start;
           try {
             const parsed = JSON.parse(body);
-            if (res.statusCode !== 200) {
+            if (res.statusCode === 503 && parsed?.error === "draining") {
+              const error = new Error(
+                `Browser pool leader is draining (${parsed.reason || "leadership-lost"}) \u2014 re-elect and retry`
+              );
+              logger.debug(`[BrowserClient] Leader draining on ${path19}; re-election required.`);
+              errorTracker.trackError(error, {
+                component: "browser-manager",
+                operation,
+                errorType: "draining"
+              });
+              reject(error);
+            } else if (res.statusCode !== 200) {
               const error = new Error(parsed.error || `HTTP ${res.statusCode}`);
               errorTracker.trackError(error, {
                 component: "browser-manager",
@@ -10777,7 +10790,10 @@ function isTransientSocketError(error) {
   err.message.includes("Worker pool is shutting down") || // Poolifier throws this when pool.execute() is called while the pool is being
   // destroyed (e.g., during scheduler restart). Treat as transient — the pool will
   // be ready for new tasks once destruction completes and a new pool is initialized.
-  err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool"));
+  err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // The leader answered 503 because it is shutting down after losing
+  // leadership. Retrying re-elects and reaches the new leader, so this is
+  // transient in exactly the same sense as a pool drain.
+  err.message.includes("Browser pool leader is draining"));
 }
 function isCloudflareBlockError(error) {
   if (!error || typeof error !== "object") return false;
@@ -10789,6 +10805,11 @@ function isTaskTimeoutError(error) {
   const err = error;
   return typeof err.message === "string" && (err.message.includes("timed out after") || err.message.includes("Search task timed out") || err.message.includes("Scrape task timed out") || err.message.includes("Health check timed out") || err.message.includes("[BrowserClient] Request to"));
 }
+function isServerDrainingError(error) {
+  if (!error || typeof error !== "object") return false;
+  const err = error;
+  return typeof err.message === "string" && err.message.includes("Browser pool leader is draining");
+}
 function isPoolShutdownError(error) {
   if (!error || typeof error !== "object") return false;
   const err = error;
@@ -10807,6 +10828,7 @@ var DEFAULT_BREAKER_CONFIG = {
   name: "BrowserPool",
   isTransientError: (error) => {
     if (isPoolShutdownError(error)) return false;
+    if (isServerDrainingError(error)) return false;
     if (isCloudflareBlockError(error)) return false;
     if (isTaskTimeoutError(error)) return false;
     return isTransientSocketError(error);
@@ -10847,6 +10869,50 @@ var BrowserServer = class {
   options;
   server = null;
   port = 0;
+  /**
+   * Live client sockets. Tracked so teardown can tell whether anyone is still
+   * attached (`hasActiveClients`) rather than guessing, and so drain has a
+   * meaningful count to report.
+   */
+  connections = /* @__PURE__ */ new Set();
+  draining = false;
+  /**
+   * Why this server stopped accepting work. Reported to the client so a log line
+   * says which teardown it hit — a lost election, an idle timeout, or process
+   * shutdown all reach the same code path but are very different incidents.
+   */
+  drainReason = "shutting-down";
+  /**
+   * Enter draining mode: every *subsequent* request is answered `503` instead of
+   * being served or left to hit a closed socket. Requests already past this
+   * check run to completion, so drain never truncates in-flight work.
+   *
+   * This exists for leadership-loss teardown. Without it a client that is
+   * mid-conversation with a leader that just lost the election waits out a
+   * connect timeout or gets a bare ECONNREFUSED/RST once the listener closes,
+   * and only then re-elects. An explicit 503 turns that silent stall into an
+   * immediate, unambiguous "this leader is gone — re-elect now".
+   *
+   * Idempotent: calling it while already draining is a no-op.
+   */
+  beginDrain(reason = "shutting-down") {
+    if (this.draining) return;
+    this.draining = true;
+    this.drainReason = reason;
+    logger.log(`[BrowserServer] Draining (${reason}) \u2014 refusing new requests with 503 (${this.connections.size} client connection(s) attached).`);
+  }
+  /** True once {@link beginDrain} has been called. */
+  isDraining() {
+    return this.draining;
+  }
+  /** Whether any client socket is currently attached. */
+  hasActiveClients() {
+    return this.connections.size > 0;
+  }
+  /** Number of attached client sockets (observability / teardown decisions). */
+  getConnectionCount() {
+    return this.connections.size;
+  }
   async start() {
     return new Promise((resolve8, reject) => {
       this.server = http3.createServer(async (req, res) => {
@@ -10859,6 +10925,14 @@ var BrowserServer = class {
           res.end(JSON.stringify({ error: "Unauthorized" }));
           return;
         }
+        if (this.draining) {
+          res.writeHead(503, {
+            "Content-Type": "application/json",
+            "Connection": "close"
+          });
+          res.end(JSON.stringify({ error: "draining", reason: this.drainReason }));
+          return;
+        }
         if (req.method !== "POST") {
           res.writeHead(405);
           res.end("Method Not Allowed");
@@ -10928,6 +11002,12 @@ var BrowserServer = class {
           }
         });
       });
+      this.server.on("connection", (socket) => {
+        this.connections.add(socket);
+        socket.on("close", () => {
+          this.connections.delete(socket);
+        });
+      });
       this.server.listen(0, "127.0.0.1", () => {
         const addr = this.server?.address();
         if (addr && typeof addr === "object") {
@@ -10950,6 +11030,7 @@ var BrowserServer = class {
       return new Promise((resolve8) => {
         this.server?.close(() => {
           this.server = null;
+          this.connections.clear();
           resolve8();
         });
       });
@@ -11394,7 +11475,7 @@ var BrowserTaskScheduler = class {
           if (this.consecutiveLeadershipMisses >= this.LEADERSHIP_MISS_THRESHOLD) {
             metrics.increment("browser_leadership_lost_total", 1);
             logger.error(`[Scheduler] Leadership threshold exceeded (${this.consecutiveLeadershipMisses} misses), shutting down pool...`);
-            await this.shutdown();
+            await this.shutdown("leadership-lost");
             return;
           }
         } else {
@@ -11422,7 +11503,7 @@ var BrowserTaskScheduler = class {
     if (this.idleTimer) clearTimeout(this.idleTimer);
     this.idleTimer = setTimeout(() => {
       logger.log("[Scheduler] Browser pool idle timeout reached, shutting down...");
-      this.shutdown().catch((err) => logger.warn("[Scheduler] Idle-timeout shutdown failed (non-fatal):", err));
+      this.shutdown("idle-timeout").catch((err) => logger.warn("[Scheduler] Idle-timeout shutdown failed (non-fatal):", err));
     }, this.IDLE_TIMEOUT_MS);
     if (this.idleTimer.unref) this.idleTimer.unref();
   }
@@ -11615,9 +11696,18 @@ var BrowserTaskScheduler = class {
     }
     return result;
   }
-  async shutdown() {
+  /**
+   * @param reason why the pool is going away. Reported verbatim to clients in the
+   *   drain 503, so a lost election, an idle timeout and a process shutdown are
+   *   distinguishable in a client-side log rather than all reading as one.
+   */
+  async shutdown(reason = "shutting-down") {
     if (this.isShuttingDown) return;
     this.isShuttingDown = true;
+    if (this.server?.hasActiveClients()) {
+      logger.log(`[Scheduler] Draining ${this.server.getConnectionCount()} client connection(s) before shutdown (${reason}).`);
+    }
+    this.server?.beginDrain(reason);
     if (this.idleTimer) {
       clearTimeout(this.idleTimer);
       this.idleTimer = null;
@@ -11771,7 +11861,7 @@ async function forceSchedulerRestart(forceClearRemoteState = false, container =
     }
     if (oldScheduler && "schedulerId" in oldScheduler && oldScheduler.schedulerId) {
       if (oldScheduler instanceof BrowserTaskScheduler) {
-        const shutdownPromise = oldScheduler.shutdown().catch((err) => {
+        const shutdownPromise = oldScheduler.shutdown("leadership-lost").catch((err) => {
           logger.warn("[Scheduler] Error during old scheduler shutdown after restart:", err);
         });
         schedulerService.setPendingShutdownPromise(shutdownPromise);
@@ -12962,7 +13052,8 @@ var ProcessLifecycleService = class {
       if (expectedStartTime !== void 0 && expectedStartTime !== null) {
         const actualStartTime = await this.getProcessStartTime(pid);
         if (actualStartTime === null) {
-          return false;
+          process.kill(pid, 0);
+          return true;
         }
         return Math.abs(actualStartTime - expectedStartTime) <= 1;
       }
@@ -14130,6 +14221,193 @@ var StatePathConfiguration = class {
 // src/infrastructure/service-initialization.ts
 init_file_lock_service();
 
+// src/infrastructure/research-run-semaphore.ts
+init_logger();
+init_metrics();
+init_file_lock_service();
+init_service_registry();
+init_service_names();
+import * as pathmod2 from "node:path";
+var DEFAULT_MAX_CONCURRENT_RUNS = 3;
+var DEFAULT_ACQUIRE_TIMEOUT_MS = 6e5;
+var POLL_DELAY_MS = 250;
+var ResearchRunCapacityError = class extends Error {
+  slots;
+  constructor(slots, maxWaitMs) {
+    super(
+      `Maximum concurrent research runs (${slots}) reached${maxWaitMs > 0 ? ` and still busy after queueing for ${Math.round(maxWaitMs / 1e3)}s` : ""}; another run is still in progress. This is temporary \u2014 retry shortly. (Raise the cap with PI_RESEARCH_MAX_CONCURRENT_RUNS, or the queue wait with PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS.)`
+    );
+    this.name = "ResearchRunCapacityError";
+    this.slots = slots;
+  }
+};
+var ResearchRunSemaphore = class {
+  name = ServiceNames.RESEARCH_RUN_SEMAPHORE;
+  lifecycle = "uninitialized" /* UNINITIALIZED */;
+  slotDir;
+  processLifecycle;
+  maxSlots;
+  defaultMaxWaitMs;
+  /** Rotating start index so contending processes don't all hammer slot 0 first. */
+  nextStartIndex = 0;
+  _initialized = false;
+  /**
+   * @param slotDir    Directory holding the `research-run-slot-<i>.lock` files.
+   * @param processLifecycle PID+startTime liveness source (PID-reuse-safe reclamation).
+   * @param maxSlots   N — maximum concurrent runs. Defaults from env then {@link DEFAULT_MAX_CONCURRENT_RUNS}.
+   * @param defaultMaxWaitMs Default acquire wait. Defaults from env then {@link DEFAULT_ACQUIRE_TIMEOUT_MS}.
+   */
+  constructor(slotDir, processLifecycle, maxSlots, defaultMaxWaitMs) {
+    this.slotDir = slotDir;
+    this.processLifecycle = processLifecycle;
+    this.maxSlots = Math.max(1, Math.floor(
+      maxSlots ?? numberFromEnv("PI_RESEARCH_MAX_CONCURRENT_RUNS") ?? DEFAULT_MAX_CONCURRENT_RUNS
+    ));
+    this.defaultMaxWaitMs = Math.max(0, Math.floor(
+      defaultMaxWaitMs ?? numberFromEnv("PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS") ?? DEFAULT_ACQUIRE_TIMEOUT_MS
+    ));
+  }
+  /** Number of slots (max concurrent runs). */
+  getMaxSlots() {
+    return this.maxSlots;
+  }
+  async initialize() {
+    if (this._initialized) return;
+    this.lifecycle = "initializing" /* INITIALIZING */;
+    try {
+      await import("node:fs/promises").then((fs17) => fs17.mkdir(this.slotDir, { recursive: true, mode: 448 }));
+    } catch (err) {
+      logger.warn(`[ResearchRunSemaphore] Could not create slot dir ${this.slotDir}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    this._initialized = true;
+    this.lifecycle = "initialized" /* INITIALIZED */;
+    logger.log(`[ResearchRunSemaphore] Initialized with ${this.maxSlots} run slot(s) (acquire timeout ${this.defaultMaxWaitMs}ms).`);
+  }
+  async dispose() {
+    this.lifecycle = "disposed" /* DISPOSED */;
+  }
+  /**
+   * Try to win one slot NON-BLOCKING. Builds a fresh {@link FileLockService}
+   * configured as a try-take (see class doc: "fresh instance per attempt").
+   * Returns an acquisition on success, or null if the slot is busy / lost a race.
+   */
+  async trySlot(slotIndex) {
+    const slot = new FileLockService({
+      lockFilePath: pathmod2.join(this.slotDir, `research-run-slot-${slotIndex}.lock`),
+      lockTimeout: 0,
+      // fail immediately on a live-contended slot (try-take)
+      lockRetries: 24,
+      // enough iterations to complete a dead-owner reclaim + reacquire
+      lockRetryDelay: 50,
+      lockStaleThreshold: 15e3,
+      // legacy (no-pid) locks: reclaim once 15s old
+      // A live process holding a run slot for the entire (minutes-long) run is the
+      // NORMAL case, not a stall. Never steal from a live owner — doing so would
+      // admit an (N+1)th concurrent run, the exact condition this guard prevents.
+      // Dead owners are reclaimed immediately regardless of age.
+      liveOwnerStaleThreshold: Number.MAX_SAFE_INTEGER,
+      processLifecycle: this.processLifecycle ?? void 0
+    });
+    try {
+      await slot.initialize();
+      await slot.acquireLock();
+    } catch {
+      return null;
+    }
+    return {
+      slotIndex,
+      release: async () => {
+        try {
+          await slot.releaseLock();
+        } catch (err) {
+          logger.warn(`[ResearchRunSemaphore] Failed to release slot ${slotIndex}: ${err instanceof Error ? err.message : String(err)}`);
+        } finally {
+          metrics.setGauge("research_run_slots_held", 0, { slot: String(slotIndex) });
+        }
+      }
+    };
+  }
+  /**
+   * Acquire one run slot, waiting up to `maxWaitMs` (defaults to the constructed
+   * value) when all slots are held by live processes. Resolves with a handle that
+   * must be released when the run ends.
+   *
+   * @param signal aborts the wait. Without it a run started under a long
+   *   `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS` would ignore Ctrl-C until the deadline
+   *   passed, because the wait happens before any orchestrator exists to cancel.
+   * @param onQueued invoked at most once, when the first full sweep finds every
+   *   slot busy and the call is about to wait. Lets a front-end tell the user the
+   *   run is queued rather than hung. Never allowed to break the acquire.
+   * @throws {ResearchRunCapacityError} if no slot frees within `maxWaitMs`.
+   * @throws {Error} named `AbortError` if `signal` aborts while waiting.
+   */
+  async acquire(maxWaitMs = this.defaultMaxWaitMs, signal, onQueued) {
+    if (!this._initialized) await this.initialize();
+    if (signal?.aborted) {
+      metrics.increment("research_run_acquire_total", 1, { status: "aborted" });
+      throwIfAborted(signal);
+    }
+    const deadline = Date.now() + Math.max(0, maxWaitMs);
+    let announcedQueued = false;
+    for (; ; ) {
+      const start = this.nextStartIndex;
+      this.nextStartIndex = (this.nextStartIndex + 1) % this.maxSlots;
+      for (let offset = 0; offset < this.maxSlots; offset++) {
+        const i = (start + offset) % this.maxSlots;
+        const got = await this.trySlot(i);
+        if (got) {
+          metrics.setGauge("research_run_slots_held", 1, { slot: String(i) });
+          metrics.increment("research_run_acquire_total", 1, { status: "acquired" });
+          return got;
+        }
+      }
+      if (Date.now() >= deadline) {
+        metrics.increment("research_run_acquire_total", 1, { status: "exhausted" });
+        throw new ResearchRunCapacityError(this.maxSlots, maxWaitMs);
+      }
+      if (!announcedQueued) {
+        announcedQueued = true;
+        logger.info(`[ResearchRunSemaphore] All ${this.maxSlots} run slot(s) busy \u2014 queueing for up to ${maxWaitMs}ms.`);
+        try {
+          onQueued?.(this.maxSlots, maxWaitMs);
+        } catch (err) {
+          logger.debug(`[ResearchRunSemaphore] onQueued callback threw: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      }
+      try {
+        await sleep(POLL_DELAY_MS, signal);
+      } catch (err) {
+        metrics.increment("research_run_acquire_total", 1, { status: "aborted" });
+        throw err;
+      }
+    }
+  }
+};
+function throwIfAborted(signal) {
+  if (signal?.aborted) {
+    throw Object.assign(new Error("Research run slot acquisition aborted"), { name: "AbortError" });
+  }
+}
+function numberFromEnv(key) {
+  const raw = process.env[key];
+  if (raw === void 0 || raw === "") return void 0;
+  const n = Number(raw);
+  return Number.isFinite(n) ? n : void 0;
+}
+function sleep(ms, signal) {
+  return new Promise((resolve8, reject) => {
+    const timer = setTimeout(() => {
+      if (onAbort) signal?.removeEventListener("abort", onAbort);
+      resolve8();
+    }, ms);
+    const onAbort = signal ? () => {
+      clearTimeout(timer);
+      reject(Object.assign(new Error("Research run slot acquisition aborted"), { name: "AbortError" }));
+    } : void 0;
+    if (onAbort) signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
 // src/infrastructure/state/state-backup-manager.ts
 init_logger();
 init_service_registry();
@@ -14389,6 +14667,19 @@ function registerInfrastructureServices(container = getServiceContainer()) {
     },
     container
   );
+  registerService(
+    ServiceNames.RESEARCH_RUN_SEMAPHORE,
+    async () => {
+      const pathConfig = await getService(ServiceNames.STATE_PATH_CONFIGURATION, void 0, container);
+      const processLifecycle = await getService(ServiceNames.PROCESS_LIFECYCLE, void 0, container);
+      return new ResearchRunSemaphore(pathConfig.getStateDir(), processLifecycle);
+    },
+    {
+      allowOverwrite: false,
+      enableLogging: true
+    },
+    container
+  );
   registerService(
     ServiceNames.STATE_BACKUP_MANAGER,
     async () => {
@@ -14608,6 +14899,17 @@ init_service_registry();
 init_service_interfaces();
 var lastRestartTime = 0;
 var RESTART_COOLDOWN_MS = 1e4;
+var LEADER_HANDOVER_RETRIES = 3;
+var HANDOVER_BACKOFF_MS = 1200;
+async function recoverFromLeaderHandover(container) {
+  logger.warn("[BrowserManager] Leader handover detected (pool draining) \u2014 dropping stale scheduler handle and re-resolving...");
+  await forceSchedulerRestart2(false, container).catch(
+    (err) => logger.debug("[BrowserManager] Scheduler re-resolve during handover failed (non-fatal):", err)
+  );
+  await waitForBrowserPoolIdle(15e3).catch(
+    (err) => logger.debug("Wait for browser idle timed out or failed:", err)
+  );
+}
 async function getScheduler2(config, container = getServiceContainer()) {
   try {
     const factory = await getService(ServiceNames.SCHEDULER_FACTORY, void 0, container);
@@ -14624,7 +14926,7 @@ async function forceSchedulerRestart2(forceClearRemoteState = false, container =
     return await forceSchedulerRestart(forceClearRemoteState, container);
   }
 }
-async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retries = 1, container = getServiceContainer()) {
+async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retries = 1, container = getServiceContainer(), handoverRetries = LEADER_HANDOVER_RETRIES) {
   if (signal?.aborted) throw new Error("Aborted");
   const sessionId = typeof taskOrUrl === "object" ? taskOrUrl.sessionId : void 0;
   const breaker2 = sessionId ? getBrowserCircuitBreaker(sessionId) : browserCircuitBreaker;
@@ -14645,6 +14947,19 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
     });
   } catch (error) {
     if (signal?.aborted || error instanceof Error && error.message === "Aborted") throw new Error("Aborted", { cause: error });
+    const isHandover = isPoolShutdownError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error);
+    if (isHandover && handoverRetries > 0) {
+      errorTracker.trackError(error, {
+        component: "browser-manager",
+        operation: type,
+        taskType: type,
+        errorType: "leader_handover"
+      });
+      logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
+      await recoverFromLeaderHandover(container);
+      await new Promise((resolve8) => setTimeout(resolve8, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
+      return runBrowserTask(taskOrUrl, type, config, signal, retries, container, handoverRetries - 1);
+    }
     if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
       errorTracker.trackError(error, {
         component: "browser-manager",
@@ -14670,7 +14985,7 @@ async function runBrowserTask(taskOrUrl, type = "scrape", config, signal, retrie
       }
       const jitter = 100 + Math.floor(Math.random() * 400);
       await new Promise((resolve8) => setTimeout(resolve8, jitter));
-      return runBrowserTask(taskOrUrl, type, config, signal, retries - 1, container);
+      return runBrowserTask(taskOrUrl, type, config, signal, retries - 1, container, handoverRetries);
     }
     throw error;
   }
@@ -14690,7 +15005,7 @@ async function runBrowserHealthCheck(config, retries = 1, signal, container = ge
       });
       logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
       if (isPoolShutdownError(error)) {
-        await waitForBrowserPoolIdle(15e3).catch((err) => logger.debug("Wait for browser idle timed out or failed:", err));
+        await recoverFromLeaderHandover(container);
       } else {
         const now = Date.now();
         if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
@@ -14709,7 +15024,7 @@ async function runBrowserHealthCheck(config, retries = 1, signal, container = ge
     throw error;
   }
 }
-async function runWorkerSearch(query, config, signal, retries = 1, sessionId, container = getServiceContainer()) {
+async function runWorkerSearch(query, config, signal, retries = 1, sessionId, container = getServiceContainer(), handoverRetries = LEADER_HANDOVER_RETRIES) {
   if (signal?.aborted) throw new Error("Aborted");
   const breaker2 = sessionId ? getBrowserCircuitBreaker(sessionId) : browserCircuitBreaker;
   try {
@@ -14720,6 +15035,19 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
     });
   } catch (error) {
     if (signal?.aborted || error instanceof Error && error.message === "Aborted") throw new Error("Aborted", { cause: error });
+    const isHandover = isPoolShutdownError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error);
+    if (isHandover && handoverRetries > 0) {
+      errorTracker.trackError(error, {
+        component: "browser-manager",
+        operation: "search",
+        query,
+        errorType: "leader_handover"
+      });
+      logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
+      await recoverFromLeaderHandover(container);
+      await new Promise((resolve8) => setTimeout(resolve8, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
+      return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
+    }
     if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
       errorTracker.trackError(error, {
         component: "browser-manager",
@@ -14743,7 +15071,7 @@ async function runWorkerSearch(query, config, signal, retries = 1, sessionId, co
       }
       const jitter = 100 + Math.floor(Math.random() * 400);
       await new Promise((resolve8) => setTimeout(resolve8, jitter));
-      return runWorkerSearch(query, config, signal, retries - 1, sessionId, container);
+      return runWorkerSearch(query, config, signal, retries - 1, sessionId, container, handoverRetries);
     }
     throw error;
   }
@@ -15538,6 +15866,34 @@ async function getSsrfSafeDispatcher() {
   }
   return cachedSsrfDispatcher;
 }
+function formatErrorWithCause(err) {
+  try {
+    const parts = [];
+    let current = err;
+    for (let depth = 0; current !== null && current !== void 0 && depth < 5; depth++) {
+      let segment;
+      if (current instanceof Error) {
+        segment = current.message || current.name;
+        const code = current.code;
+        if (typeof code === "string" && code && !segment.includes(code)) {
+          segment = `${segment} [${code}]`;
+        }
+      } else if (typeof current === "string") {
+        segment = current;
+      } else {
+        segment = String(current);
+      }
+      if (segment && !parts.includes(segment)) parts.push(segment);
+      const next = current.cause;
+      if (next === current) break;
+      current = next;
+    }
+    if (parts.length === 0) return String(err);
+    return parts.length === 1 ? parts[0] : parts.join(" \u2190 ");
+  } catch {
+    return String(err);
+  }
+}
 function isPrivateIp(ip) {
   const parts = ip.split(".").map(Number);
   if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
@@ -15641,8 +15997,10 @@ function validateContent(html, markdown, url) {
 }
 
 // src/web-research/web-scraper.ts
+init_retry_utils();
 init_safe_unref();
 init_service_registry();
+var FETCH_RETRY_DELAY_MS = 250;
 var playwrightAvailable = false;
 var markdownConverterPromise = null;
 function initScraperDependencies() {
@@ -15779,6 +16137,17 @@ async function readBodyCapped(response, maxBytes) {
   }
   return out;
 }
+function trackFetchLayerFailure(url, error) {
+  const isSsrf = error instanceof Error && error.message.includes("not allowed");
+  errorTracker.trackError(error instanceof Error ? error : String(error), {
+    component: "scrapers",
+    operation: "fetch",
+    url,
+    domain: extractDomain(url),
+    layer: "fetch",
+    ...isSsrf ? { errorType: "ssrf_blocked" } : {}
+  });
+}
 async function scrapeWithFetch(url, signal) {
   await validateUrlForSSRF(url);
   const dispatcher = await getSsrfSafeDispatcher();
@@ -15885,14 +16254,6 @@ async function scrapeWithFetch(url, signal) {
         const sizeMB = Math.round(e.bytesRead / 1024 / 1024);
         logger.warn(`[Scrapers] HTML response too large (streamed ${sizeMB}MB, max 25MB), skipping`);
         metrics.increment("scrape_errors_total", 1, { error_type: "size_exceeded", content_type: "html" });
-        errorTracker.trackError(new Error(`HTML response too large (${sizeMB}MB, max 25MB)`), {
-          component: "scrapers",
-          operation: "fetch",
-          url,
-          domain: extractDomain(url),
-          layer: "fetch",
-          contentType: "html"
-        });
         throw new Error(`HTML response too large (${sizeMB}MB, max 25MB)`, { cause: e });
       }
       throw e;
@@ -15901,14 +16262,6 @@ async function scrapeWithFetch(url, signal) {
       const sizeMB = Math.round(bodyBytes.length / 1024 / 1024);
       logger.warn(`[Scrapers] HTML response too large (actual: ${sizeMB}MB, max 25MB), skipping`);
       metrics.increment("scrape_errors_total", 1, { error_type: "size_exceeded", content_type: "html" });
-      errorTracker.trackError(new Error(`HTML response too large (${sizeMB}MB, max 25MB)`), {
-        component: "scrapers",
-        operation: "fetch",
-        url,
-        domain: extractDomain(url),
-        layer: "fetch",
-        contentType: "html"
-      });
       throw new Error(`HTML response too large (${sizeMB}MB, max 25MB)`);
     }
     const html = decodeHtmlBody(bodyBytes, contentType);
@@ -15927,24 +16280,9 @@ async function scrapeWithFetch(url, signal) {
     if (error instanceof Error && error.message.includes("not allowed")) {
       metrics.increment("scrape_operations_total", 1, { layer: "fetch", status: "ssrf_blocked" });
       metrics.observe("scrape_latency_ms", Date.now() - fetchStart, { layer: "fetch", status: "ssrf_blocked" });
-      errorTracker.trackError(error, {
-        component: "scrapers",
-        operation: "fetch",
-        url,
-        domain: extractDomain(url),
-        layer: "fetch",
-        errorType: "ssrf_blocked"
-      });
       throw error;
     }
     metrics.increment("scrape_operations_total", 1, { layer: "fetch", status: "error" });
-    errorTracker.trackError(error instanceof Error ? error : String(error), {
-      component: "scrapers",
-      operation: "fetch",
-      url,
-      domain: extractDomain(url),
-      layer: "fetch"
-    });
     throw error;
   } finally {
     clearTimeout(timeoutId);
@@ -16017,7 +16355,21 @@ async function scrapeSingle(url, signal, config, sessionId, container = getServi
   }
   const start = Date.now();
   try {
-    const res = await scrapeWithFetch(url, signal);
+    const res = await (async () => {
+      try {
+        return await scrapeWithFetch(url, signal);
+      } catch (firstErr) {
+        if (!isTransientError(firstErr) || signal?.aborted) throw firstErr;
+        logger.debug(`[Scrapers] fetch transient failure for ${url}, retrying once: ${formatErrorWithCause(firstErr)}`);
+        metrics.increment("scrape_fetch_retries_total", 1, { layer: "fetch" });
+        try {
+          await abortableDelay(FETCH_RETRY_DELAY_MS, signal, true);
+        } catch {
+          throw firstErr;
+        }
+        return await scrapeWithFetch(url, signal);
+      }
+    })();
     const duration = Date.now() - start;
     logger.log(`[Scrapers] fetch success for ${url} in ${duration}ms`);
     metrics.increment("scrape_results_total", 1, { outcome: "fetch_success" });
@@ -16027,7 +16379,8 @@ async function scrapeSingle(url, signal, config, sessionId, container = getServi
     return { ...res, url, success: true };
   } catch (e1) {
     const fetchDuration = Date.now() - start;
-    logger.debug(`[Scrapers] fetch failed for ${url} in ${fetchDuration}ms: ${String(e1)}`);
+    logger.debug(`[Scrapers] fetch failed for ${url} in ${fetchDuration}ms: ${formatErrorWithCause(e1)}`);
+    trackFetchLayerFailure(url, e1);
     if (playwrightAvailable) {
       try {
         const browserStart = Date.now();
@@ -16044,18 +16397,18 @@ async function scrapeSingle(url, signal, config, sessionId, container = getServi
       } catch (e2) {
         const totalDuration = Date.now() - start;
         if (isBenignScrapeFailure(e2)) {
-          logger.debug(`[Scrapers] Browser fallback did not yield content for ${url} in ${totalDuration}ms: ${String(e2)}`);
+          logger.debug(`[Scrapers] Browser fallback did not yield content for ${url} in ${totalDuration}ms: ${formatErrorWithCause(e2)}`);
         } else {
           logger.error(`[Scrapers] Browser fallback failed for ${url} in ${totalDuration}ms:`, e2);
         }
         metrics.increment("scrape_errors_total", 1, { error_type: "fallback_failed", layer: "playwright" });
         metrics.increment("scrape_results_total", 1, { outcome: "total_failure" });
-        return { url, success: false, error: String(e2), markdown: "" };
+        return { url, success: false, error: formatErrorWithCause(e2), markdown: "" };
       }
     }
     metrics.increment("scrape_errors_total", 1, { error_type: "no_fallback_available", layer: "fetch" });
     metrics.increment("scrape_results_total", 1, { outcome: "total_failure" });
-    return { url, success: false, error: String(e1), markdown: "" };
+    return { url, success: false, error: formatErrorWithCause(e1), markdown: "" };
   }
 }
 async function scrape(urls, maxConcurrency = 5, signal, config, sessionId, onUrlComplete, container = getServiceContainer()) {
@@ -20510,6 +20863,9 @@ var HeadlessObserver = class {
   onStart(query, complexity) {
     this.emit("start", { query, complexity });
   }
+  onRunQueued(slots, maxWaitMs) {
+    this.emit("run_queued", { slots, maxWaitMs });
+  }
   onPlanningStart(attempt) {
     this.emit("planning_start", { attempt });
   }
@@ -21211,6 +21567,8 @@ var DeepResearchOrchestrator = class {
       } catch (obsErr) {
         logger.debug("[DeepOrchestrator] onComplete observer threw:", obsErr);
       }
+      logger.debug(`[DeepOrchestrator] Final synthesis for ${researchId} (${result.length} chars):
+${result}`);
       return result;
     } catch (error) {
       const sessionDuration = Date.now() - this.sessionStart;
@@ -21234,6 +21592,8 @@ var DeepResearchOrchestrator = class {
           } catch (obsErr) {
             logger.debug("[DeepOrchestrator] fallback onComplete observer threw:", obsErr);
           }
+          logger.debug(`[DeepOrchestrator] Fallback synthesis for ${researchId} (${result.length} chars):
+${result}`);
           return result;
         }
       } catch (fallbackErr) {
@@ -21300,6 +21660,19 @@ var ResearchOrchestrationService = class {
     const selectedModel = await this.resolveResearchModel(options);
     logger.info(`[ResearchOrchestrationService] Using model: ${selectedModel.provider}/${selectedModel.id}`);
     const researchStart = Date.now();
+    let runSlot = null;
+    try {
+      const container = tryGetServiceContainerFromCtx(ctx);
+      const semaphore = await getService(ServiceNames.RESEARCH_RUN_SEMAPHORE, ctx, container);
+      runSlot = await semaphore.acquire(void 0, signal, (slots, maxWaitMs) => {
+        notifyRunQueued(observer, slots, maxWaitMs);
+      });
+      logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
+    } catch (err) {
+      if (err instanceof ResearchRunCapacityError) throw err;
+      if (err instanceof Error && err.name === "AbortError") throw err;
+      logger.warn(`[ResearchOrchestrationService] Run-cap unavailable, proceeding without it: ${err instanceof Error ? err.message : String(err)}`);
+    }
     let result;
     try {
       if (depth === 0) {
@@ -21342,6 +21715,14 @@ var ResearchOrchestrationService = class {
       metrics.observe("research_manager_latency_ms", researchDuration, { depth: String(depth), status: "error", source: "extension" });
       metrics.increment("research_manager_requests_total", 1, { depth: String(depth), status: "error", source: "extension" });
       throw error;
+    } finally {
+      if (runSlot) {
+        await runSlot.release().catch((relErr) => {
+          logger.warn(
+            `[ResearchOrchestrationService] Failed to release run slot ${runSlot?.slotIndex}; capacity is reduced until this process exits: ${relErr instanceof Error ? relErr.message : String(relErr)}`
+          );
+        });
+      }
     }
   }
   /**
@@ -21639,6 +22020,15 @@ var ResearchOrchestrationService = class {
     }
   }
 };
+function notifyRunQueued(observer, slots, maxWaitMs) {
+  try {
+    const obs = observer;
+    if (typeof obs?.onRunQueued === "function") obs.onRunQueued(slots, maxWaitMs);
+    else if (typeof obs?.onProgress === "function") obs.onProgress("run_queued", { slots, maxWaitMs });
+  } catch (err) {
+    logger.debug(`[ResearchOrchestrationService] run_queued notification threw: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
 
 // src/orchestration/research-session-service.ts
 init_logger();
@@ -21782,10 +22172,63 @@ var ResearchSessionService = class {
 // src/orchestration/research-synthesis-service.ts
 init_text_utils();
 init_text_utils();
+init_url_utils();
 init_logger();
 init_service_registry();
 init_service_names();
 var NO_SOURCES_NOTE = "CITED LINKS\n_No web sources were successfully retrieved for this report \u2014 searches or page fetches may have been blocked (for example by site bot-protection) or returned no usable results. Treat the findings above as unverified._";
+var UNVERIFIED_LINKS_NOTE = "_None of the links above could be verified: no page fetch succeeded this session and no citation could be matched to retrieved content. They may be inaccurate or invented \u2014 check each one before relying on it._";
+var MAX_PLAUSIBLE_CITATION_MARKER = 200;
+function rewriteCitationMarkers(text, remap, isDangling) {
+  const inFence = fencedCodeMask(text);
+  return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad, digits, offset) => {
+    const n = parseInt(digits, 10);
+    const mapped = remap.get(n);
+    if (mapped !== void 0) return `${pad}[${mapped}]`;
+    if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
+    if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
+    const lineStart = text.lastIndexOf("\n", offset - 1) + 1;
+    return text.slice(lineStart, offset).trim() === "" ? pad : "";
+  });
+}
+function fencedCodeMask(text) {
+  const ranges = [];
+  let open4 = -1;
+  let pos = 0;
+  for (const line of text.split("\n")) {
+    if (/^\s{0,3}(```|~~~)/.test(line)) {
+      if (open4 < 0) open4 = pos;
+      else {
+        ranges.push([open4, pos + line.length]);
+        open4 = -1;
+      }
+    }
+    pos += line.length + 1;
+  }
+  if (open4 >= 0) ranges.push([open4, text.length]);
+  return (offset) => ranges.some(([start, end]) => offset >= start && offset < end);
+}
+function inInlineCodeSpan(text, offset) {
+  const lineStart = text.lastIndexOf("\n", offset - 1) + 1;
+  let ticks = 0;
+  for (let i = lineStart; i < offset; i++) if (text[i] === "`") ticks++;
+  return ticks % 2 === 1;
+}
+function redactUnverifiedProseUrls(text, trusted) {
+  if (trusted.size === 0) return text;
+  let removed = 0;
+  const result = text.replace(/https?:\/\/[^\s<>"']+/gi, (token) => {
+    const core = stripTrailingLlmPunctuation(token);
+    const tail = token.slice(core.length);
+    if (!core || trusted.has(normalizeUrl(core))) return token;
+    removed++;
+    return `[link removed: not found in retrieved sources]${tail}`;
+  });
+  if (removed > 0) {
+    logger.warn(`[ResearchSynthesisService] Redacted ${removed} unverified inline URL(s) from synthesis prose (not present in retrieved sources).`);
+  }
+  return result;
+}
 var ResearchSynthesisService = class _ResearchSynthesisService {
   name = ServiceNames.RESEARCH_SYNTHESIS_SERVICE;
   lifecycle = "uninitialized" /* UNINITIALIZED */;
@@ -21939,35 +22382,42 @@ Research performed using ${modelId}`;
     }
     if (globalCitations.length > 0) {
       const verifiedLinksSection = formatCitedLinks(globalCitations);
+      const trustedUrls = new Set(globalCitations.map((c) => normalizeUrl(c.url)));
       const citedHeaderIdx = lastCitedLinksHeaderIndex(synthesis);
       if (citedHeaderIdx >= 0) {
         logger.debug("[ResearchSynthesisService] Replacing existing CITED LINKS with verified version");
         const ownCitations = parseCitations(synthesis);
         const urlToGlobalId = new Map(globalCitations.map((c) => [normalizeUrl(c.url), c.id]));
         const localToGlobal = /* @__PURE__ */ new Map();
+        const droppedNumbers = /* @__PURE__ */ new Set();
         ownCitations.forEach((cit, index) => {
+          const localKey = cit.number ?? index + 1;
           const globalId = urlToGlobalId.get(normalizeUrl(cit.url));
-          if (globalId !== void 0) localToGlobal.set(cit.number ?? index + 1, globalId);
+          if (globalId !== void 0) localToGlobal.set(localKey, globalId);
+          else droppedNumbers.add(localKey);
         });
         let body = synthesis.slice(0, citedHeaderIdx).trimEnd();
-        const needsRemap = [...localToGlobal].some(([local, global]) => local !== global);
-        if (needsRemap) {
-          body = body.replace(/\[(\d+)\]/g, (match, p1) => {
-            const globalId = localToGlobal.get(parseInt(p1, 10));
-            return globalId !== void 0 ? `[${globalId}]` : match;
-          });
+        if (localToGlobal.size > 0 || droppedNumbers.size > 0) {
+          body = rewriteCitationMarkers(body, localToGlobal, (n) => droppedNumbers.has(n));
         }
+        body = redactUnverifiedProseUrls(body, trustedUrls);
         return `${body}
 
 ${verifiedLinksSection}`;
       }
       logger.warn("[ResearchSynthesisService] Synthesis missing CITED LINKS - appending verified version");
-      return `${synthesis.trim()}
+      const validIds = new Set(globalCitations.map((c) => c.id));
+      let prose = rewriteCitationMarkers(synthesis.trim(), /* @__PURE__ */ new Map(), (n) => !validIds.has(n));
+      prose = redactUnverifiedProseUrls(prose, trustedUrls);
+      return `${prose}
 
 ${verifiedLinksSection}`;
     }
     if (/CITED LINKS/i.test(synthesis)) {
-      return synthesis;
+      logger.warn("[ResearchSynthesisService] Preserving an unparseable CITED LINKS block from a run that retrieved no sources \u2014 flagging it as unverified.");
+      return `${synthesis.trimEnd()}
+
+${UNVERIFIED_LINKS_NOTE}`;
     }
     logger.warn("[ResearchSynthesisService] No citations and no scrape provenance \u2014 appending explicit no-sources note.");
     return `${synthesis.trim()}
@@ -22669,7 +23119,7 @@ init_logger();
 init_constants();
 import { promises as fs14 } from "node:fs";
 import { homedir as homedir4, tmpdir as tmpdir4, platform as platform3 } from "node:os";
-import { join as join17, resolve as resolve3, normalize, sep as sep2 } from "node:path";
+import { join as join18, resolve as resolve3, normalize, sep as sep2 } from "node:path";
 import { randomBytes as randomBytes5 } from "node:crypto";
 var PREFERRED_SUBDIRS = [
   "research",
@@ -22740,7 +23190,7 @@ async function findPreferredSubdir(cwd) {
   for (const target of PREFERRED_SUBDIRS) {
     const match = entries.find((e) => e.toLowerCase() === target);
     if (match === void 0) continue;
-    const candidate = join17(cwd, match);
+    const candidate = join18(cwd, match);
     try {
       const stat5 = await fs14.stat(candidate);
       if (stat5.isDirectory()) return candidate;
@@ -22785,7 +23235,7 @@ async function exportResearchReport(query, result, _mode, cwd, explicitDir) {
   for (let attempt = 0; attempt < MAX_EXPORT_RETRIES; attempt++) {
     const hash = generateHash();
     const filename = `${baseFilename}-${hash}.md`;
-    const filepath = join17(targetDir, filename);
+    const filepath = join18(targetDir, filename);
     try {
       await fs14.writeFile(filepath, result, { flag: "wx" });
       logger.log(`[export] Research report saved to: ${filepath}`);
@@ -22835,7 +23285,15 @@ var EXIT = {
   /** Missing setup: package, model, or API key not configured. */
   CONFIG: 78,
   /** Something went wrong at runtime (network, provider, internal). */
-  SOFTWARE: 70
+  SOFTWARE: 70,
+  /**
+   * Temporary failure — the request was well-formed and the install is healthy,
+   * but the machine is at capacity right now (all research run slots held by
+   * live runs). Distinct from SOFTWARE so a calling agent can tell "busy, try
+   * again shortly" apart from "this run crashed" and back off instead of
+   * re-running a broken command.
+   */
+  TEMPFAIL: 75
 };
 function toStderr(s) {
   process.stderr.write(s);
@@ -22855,6 +23313,12 @@ function makeProgressObserver() {
           toStderr(`  \u2022 plan: ${data?.plan?.researchers?.length ?? "?"} researcher(s)
 `);
           break;
+        case "run_queued":
+          toStderr(
+            `  \u2022 queued: ${data?.slots ?? "?"} concurrent run(s) already active \u2014 waiting up to ${Math.round((data?.maxWaitMs ?? 0) / 1e3)}s for a slot\u2026
+`
+          );
+          break;
         case "round_start":
           toStderr(`  \u2022 round ${data?.round}
 `);
@@ -23399,17 +23863,25 @@ async function reportError(err, what, json) {
   const msg2 = err instanceof Error ? err.message : String(err);
   const lower = msg2.toLowerCase();
   const isTaggedResearchStop = err?.code === RESEARCH_STOPPED_ERROR_CODE;
-  const isConfigError = !isTaggedResearchStop && (lower.includes("no llm model available") || lower.includes("not found in pi") || lower.includes("no api key") || lower.includes("provider must be specified") || lower.includes("invalid model string") || lower.includes("authentication") || lower.includes("unauthorized") || lower.includes("api key"));
+  const isCapacity = err instanceof Error && err.name === "ResearchRunCapacityError";
+  const isConfigError = !isTaggedResearchStop && !isCapacity && (lower.includes("no llm model available") || lower.includes("not found in pi") || lower.includes("no api key") || lower.includes("provider must be specified") || lower.includes("invalid model string") || lower.includes("authentication") || lower.includes("unauthorized") || lower.includes("api key"));
   const isRateLimit = lower.includes("429") || lower.includes("rate limit") || lower.includes("too many requests");
-  const exitCode = isConfigError && !isRateLimit ? EXIT.CONFIG : EXIT.SOFTWARE;
+  const exitCode = isCapacity ? EXIT.TEMPFAIL : isConfigError && !isRateLimit ? EXIT.CONFIG : EXIT.SOFTWARE;
   if (json) {
     const payload = { ok: false, error: msg2, exitCode };
+    if (isCapacity || isRateLimit) payload["retryable"] = true;
     if (process.env["PI_RESEARCH_DEBUG"] === "true" && err instanceof Error && err.stack) {
       payload["stack"] = err.stack;
     }
     toStdout(pretty(payload));
     return exitCode;
   }
+  if (isCapacity) {
+    toStderr(`
+Error: pi-research ${what} did not start: ${msg2}
+`);
+    return exitCode;
+  }
   if (isRateLimit) {
     toStderr(`
 Error: pi-research ${what} halted: provider rate limit reached. Wait a moment and retry.