@lincoln504/pi-research 1.0.10 → 1.1.0

package/src/infrastructure/browser/task-execution-service.ts

@@ -22,6 +22,46 @@ import type { ISchedulerFactory, IScheduler } from '../../core/scheduler-factory
 let lastRestartTime = 0;
 const RESTART_COOLDOWN_MS = 10000;
 
+/**
+ * Extra attempts reserved for a *leader handover* — the elected browser-pool leader
+ * exiting while this process still has work in flight ("Worker pool is shutting
+ * down" / "draining").
+ *
+ * This has its own budget because it is not a random socket blip: it is a known,
+ * self-healing window that lasts as long as a fresh election takes (measured at
+ * ~4 s on a loaded machine), and it hits *every* in-flight task of the affected
+ * run at once. With only the single generic retry, a whole 20-query search burst
+ * could be wiped out by one leader exiting — the run then failed outright with
+ * "all N queries encountered worker errors", seconds before the new leader came up.
+ * Spending these attempts is strictly cheaper than losing the run.
+ */
+const LEADER_HANDOVER_RETRIES = 3;
+/** Backoff between handover attempts — long enough for an election to complete. */
+const HANDOVER_BACKOFF_MS = 1200;
+
+/**
+ * Recover from a leader handover before retrying.
+ *
+ * Critically this DROPS the cached scheduler handle. A pool-shutdown error means
+ * the leader we are connected to is going away, so the cached client points at a
+ * corpse; simply waiting and retrying reconnects to the same dead port (observed
+ * as a retry logging "Connecting to existing scheduler" with the *same* version id
+ * and failing identically). Clearing it forces the next `getScheduler()` to
+ * re-resolve — joining the new leader, or winning the election itself.
+ *
+ * `forceSchedulerRestart(false)` keeps its own liveness probe and in-progress
+ * guard, so concurrent callers coalesce instead of stampeding.
+ */
+async function recoverFromLeaderHandover(container: ServiceContainer): Promise<void> {
+    logger.warn('[BrowserManager] Leader handover detected (pool draining) — dropping stale scheduler handle and re-resolving...');
+    await forceSchedulerRestart(false, container).catch((err) =>
+        logger.debug('[BrowserManager] Scheduler re-resolve during handover failed (non-fatal):', err),
+    );
+    await waitForBrowserPoolIdle(15000).catch((err) =>
+        logger.debug('Wait for browser idle timed out or failed:', err),
+    );
+}
+
 /**
  * Get the scheduler from the factory, optionally using a specific container.
  */
@@ -56,7 +96,8 @@ export async function runBrowserTask<T>(
     config?: Config,
     signal?: AbortSignal,
     retries = 1,
-    container: ServiceContainer = getServiceContainer()
+    container: ServiceContainer = getServiceContainer(),
+    handoverRetries = LEADER_HANDOVER_RETRIES,
 ): Promise<T> {
     if (signal?.aborted) throw new Error('Aborted');
 
@@ -84,6 +125,23 @@ export async function runBrowserTask<T>(
     } catch (error: unknown) {
         if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw new Error('Aborted', { cause: error });
 
+        // A leader handover gets its own budget: it is a recoverable, seconds-long
+        // window that strikes every in-flight task at once, so it must not be
+        // charged against (or limited by) the single generic transient retry.
+        const isHandover = isPoolShutdownError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error);
+        if (isHandover && handoverRetries > 0) {
+            errorTracker.trackError(error, {
+                component: 'browser-manager',
+                operation: type,
+                taskType: type,
+                errorType: 'leader_handover',
+            });
+            logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
+            await recoverFromLeaderHandover(container);
+            await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
+            return runBrowserTask<T>(taskOrUrl, type, config, signal, retries, container, handoverRetries - 1);
+        }
+
         if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
             errorTracker.trackError(error, {
                 component: 'browser-manager',
@@ -92,9 +150,9 @@ export async function runBrowserTask<T>(
                 errorType: 'transient_socket_error',
             });
             logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
-            
+
             if (isPoolShutdownError(error)) {
-                // Pool is temporarily draining — wait for the drain to finish.
+                // Handover budget already spent — fall back to a plain drain wait.
                 logger.warn(`[BrowserManager] Pool is draining — waiting for pool idle before retry...`);
                 await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
             } else {
@@ -120,7 +178,7 @@ export async function runBrowserTask<T>(
             // Retry with jitter (100-500ms) to prevent thundering herd
             const jitter = 100 + Math.floor(Math.random() * 400);
             await new Promise(resolve => setTimeout(resolve, jitter));
-            return runBrowserTask<T>(taskOrUrl, type, config, signal, retries - 1, container);
+            return runBrowserTask<T>(taskOrUrl, type, config, signal, retries - 1, container, handoverRetries);
         }
         throw error;
     }
@@ -143,9 +201,12 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
                 errorType: 'transient_socket_error',
             });
             logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
-            
+
             if (isPoolShutdownError(error)) {
-                await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
+                // Same stale-handle trap as the task paths: waiting alone leaves the
+                // cached client aimed at the departing leader, so the retry re-probes
+                // a corpse and reports the pool unhealthy during a routine handover.
+                await recoverFromLeaderHandover(container);
             } else {
                 // See runBrowserTask: don't force-clear remote leader state on a bare
                 // transient socket error — let the liveness probe inside
@@ -172,7 +233,7 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
 /**
  * Run a worker search query with retry logic.
  */
-export async function runWorkerSearch(query: string, config?: Config, signal?: AbortSignal, retries = 1, sessionId?: string, container: ServiceContainer = getServiceContainer()): Promise<SearchResult[]> {
+export async function runWorkerSearch(query: string, config?: Config, signal?: AbortSignal, retries = 1, sessionId?: string, container: ServiceContainer = getServiceContainer(), handoverRetries = LEADER_HANDOVER_RETRIES): Promise<SearchResult[]> {
     if (signal?.aborted) throw new Error('Aborted');
     
     const breaker = sessionId ? getBrowserCircuitBreaker(sessionId) : browserCircuitBreaker;
@@ -186,6 +247,22 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
     } catch (error: unknown) {
         if (signal?.aborted || (error instanceof Error && error.message === 'Aborted')) throw new Error('Aborted', { cause: error });
 
+        // See runBrowserTask: a leader handover wipes out an entire search burst at
+        // once, so it gets its own attempts rather than the single generic retry.
+        const isHandover = isPoolShutdownError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error);
+        if (isHandover && handoverRetries > 0) {
+            errorTracker.trackError(error, {
+                component: 'browser-manager',
+                operation: 'search',
+                query,
+                errorType: 'leader_handover',
+            });
+            logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
+            await recoverFromLeaderHandover(container);
+            await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
+            return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
+        }
+
         if (retries > 0 && isTransientSocketError(error) && !isTaskTimeoutError(error) && !isCloudflareBlockError(error)) {
             errorTracker.trackError(error, {
                 component: 'browser-manager',
@@ -194,7 +271,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
                 errorType: 'transient_socket_error',
             });
             logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${(error instanceof Error ? error.message : String(error)).substring(0, 100)}...`);
-            
+
             if (isPoolShutdownError(error)) {
                 await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
             } else {
@@ -214,7 +291,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
 
             const jitter = 100 + Math.floor(Math.random() * 400);
             await new Promise(resolve => setTimeout(resolve, jitter));
-            return runWorkerSearch(query, config, signal, retries - 1, sessionId, container);
+            return runWorkerSearch(query, config, signal, retries - 1, sessionId, container, handoverRetries);
         }
         throw error;
     }

package/src/infrastructure/browser/thread-worker.mjs

@@ -1356,7 +1356,10 @@ function isTransientSocketError(error) {
   err.message.includes("Worker pool is shutting down") || // Poolifier throws this when pool.execute() is called while the pool is being
   // destroyed (e.g., during scheduler restart). Treat as transient — the pool will
   // be ready for new tasks once destruction completes and a new pool is initialized.
-  err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool"));
+  err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // The leader answered 503 because it is shutting down after losing
+  // leadership. Retrying re-elects and reaches the new leader, so this is
+  // transient in exactly the same sense as a pool drain.
+  err.message.includes("Browser pool leader is draining"));
 }
 function isCloudflareBlockError(error) {
   if (!error || typeof error !== "object") return false;
@@ -1368,6 +1371,11 @@ function isTaskTimeoutError(error) {
   const err = error;
   return typeof err.message === "string" && (err.message.includes("timed out after") || err.message.includes("Search task timed out") || err.message.includes("Scrape task timed out") || err.message.includes("Health check timed out") || err.message.includes("[BrowserClient] Request to"));
 }
+function isServerDrainingError(error) {
+  if (!error || typeof error !== "object") return false;
+  const err = error;
+  return typeof err.message === "string" && err.message.includes("Browser pool leader is draining");
+}
 function isPoolShutdownError(error) {
   if (!error || typeof error !== "object") return false;
   const err = error;
@@ -1389,6 +1397,7 @@ var init_browser_error_utils = __esm({
       name: "BrowserPool",
       isTransientError: (error) => {
         if (isPoolShutdownError(error)) return false;
+        if (isServerDrainingError(error)) return false;
         if (isCloudflareBlockError(error)) return false;
         if (isTaskTimeoutError(error)) return false;
         return isTransientSocketError(error);
@@ -1404,6 +1413,7 @@ __export(scraper_utils_exports, {
   createJsMarkdownConverter: () => createJsMarkdownConverter,
   createNativeMarkdownConverter: () => createNativeMarkdownConverter,
   extractDomain: () => extractDomain,
+  formatErrorWithCause: () => formatErrorWithCause,
   getRandomUserAgent: () => getRandomUserAgent,
   getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
   isBenignScrapeFailure: () => isBenignScrapeFailure,
@@ -1535,6 +1545,34 @@ async function getSsrfSafeDispatcher() {
   }
   return cachedSsrfDispatcher;
 }
+function formatErrorWithCause(err) {
+  try {
+    const parts = [];
+    let current = err;
+    for (let depth = 0; current !== null && current !== void 0 && depth < 5; depth++) {
+      let segment;
+      if (current instanceof Error) {
+        segment = current.message || current.name;
+        const code = current.code;
+        if (typeof code === "string" && code && !segment.includes(code)) {
+          segment = `${segment} [${code}]`;
+        }
+      } else if (typeof current === "string") {
+        segment = current;
+      } else {
+        segment = String(current);
+      }
+      if (segment && !parts.includes(segment)) parts.push(segment);
+      const next = current.cause;
+      if (next === current) break;
+      current = next;
+    }
+    if (parts.length === 0) return String(err);
+    return parts.length === 1 ? parts[0] : parts.join(" \u2190 ");
+  } catch {
+    return String(err);
+  }
+}
 function isPrivateIp(ip) {
   const parts = ip.split(".").map(Number);
   if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;

package/src/infrastructure/process-lifecycle-service.ts

@@ -119,9 +119,27 @@ export class ProcessLifecycleService implements IProcessLifecycle {
       if (expectedStartTime !== undefined && expectedStartTime !== null) {
         const actualStartTime = await this.getProcessStartTime(pid);
         if (actualStartTime === null) {
-          // If we can't get start time but expected it, we can't be sure it's the same process
-          // On Linux this usually means the process just exited between the signal and the read.
-          return false;
+          // signal(0) above confirmed a live process at this PID, but we couldn't
+          // resolve its start time to verify it's the SAME instance (the PID-reuse
+          // guard). The two failure modes collapse to the same null here but must
+          // NOT be treated identically:
+          //   • Linux: usually the process exited in the microsecond gap between
+          //     signal(0) and the /proc/{pid}/stat read → genuinely dead.
+          //   • Windows/macOS: far more often the start-time lookup subprocess
+          //     (powershell `Get-Process` / `ps`) simply failed or hit its timeout
+          //     while the process is still very much alive → NOT dead.
+          // Reporting that confirmed-alive PID as DEAD is catastrophic for every
+          // caller — a live lock/leader/run-cap owner is reclaimed from under it
+          // → two writers collide (lost update) or the run-cap admits extra
+          // concurrent runs (the flaky duplicate-slot/cap-breach we saw on Windows
+          // CI). So re-confirm liveness instead of guessing: if signal(0) STILL
+          // succeeds the process is alive → return true; if it now throws it died
+          // between the two probes and the outer catch returns false. A genuine
+          // death is therefore still detected within one poll tick (~100ms); the
+          // only behavioural change is that a confirmed-alive PID is never again
+          // misreported as dead on a transient start-time lookup failure.
+          process.kill(pid, 0); // re-confirm; a throw (process died) → outer catch → false
+          return true;
         }
         // Allow ±1s slack. The macOS/BSD fallback derives start time as
         // floor(Date.now()/1000) - `ps etimes` (elapsed whole seconds); both terms are

package/src/infrastructure/research-run-semaphore.ts

@@ -0,0 +1,323 @@
+/**
+ * Research Run Semaphore
+ *
+ * Cross-process cap on the number of concurrent research runs, built on the
+ * verified {@link FileLockService}. This is the Phase 1 "run-cap" mitigation for
+ * the parallel-runs degradation documented in
+ * INVESTIGATION-2026-08-02-parallel-runs-sigabrt.md: when more than N research
+ * runs pile onto a single leader-elected browser/embedding pool, the shared
+ * priority queue saturates, runs churn through leader elections, and research
+ * degrades into ~60-error failures. Capping concurrency at the orchestration
+ * entry stops that at the source.
+ *
+ * ## Design
+ *
+ * A counting semaphore of N slots is realized as N well-known slot files
+ * (`research-run-slot-<i>.lock`) in the shared state directory, coordinated with
+ * {@link FileLockService}.
+ *
+ * - **Acquire** tries each slot once (round-robin from a rotating start index to
+ *   spread contention). Each per-slot attempt builds a **fresh** `FileLockService`
+ *   configured as a *try-take* (`lockTimeout: 0`): it either wins the slot (free,
+ *   or reclaimed from a *dead* owner) immediately or throws as soon as it sees a
+ *   *live* owner. If every slot is held by a live process, the loop waits
+ *   `pollDelayMs` and retries until `maxWaitMs` elapses (by default
+ *   {@link DEFAULT_ACQUIRE_TIMEOUT_MS} — extra runs **queue** behind the in-flight
+ *   ones instead of failing), then rejects with {@link ResearchRunCapacityError}
+ *   rather than degrading into a 60-error run.
+ * - **Release** unlinks the slot file via the captured slot service.
+ * - **Crash recovery** is automatic and PID-reuse-safe: a slot held by a process
+ *   that has died (killed, crashed, SIGKILLed by earlyoom) is reclaimed
+ *   immediately on the next acquire attempt, because `FileLockService` checks the
+ *   owner's PID+startTime liveness before treating a lock as stale.
+ *
+ * ### Why a *fresh* FileLockService per attempt
+ * `FileLockService.acquireLock()` serializes callers on a per-instance FIFO queue
+ * and only releases that queue turn in `releaseLock()` — i.e. after the run ends.
+ * Reusing a persistent per-slot instance would therefore **deadlock concurrent
+ * research starts within one process**: the first holder keeps slot[i]'s queue
+ * turn for minutes, blocking a second acquire on that slot forever (it would
+ * never reach the other slots). A throwaway instance per attempt has an empty
+ * queue, so concurrent attempts never block on stale queue state — while still
+ * reusing all of FileLockService's verified atomic-create / liveness / reclaim
+ * logic. Two concurrent attempts on the *same* slot simply race on `fs.open(wx)`:
+ * one wins, the other reads a live owner and fails fast to the next slot.
+ *
+ * ### Why `liveOwnerStaleThreshold` is effectively infinite
+ * A research run legitimately holds its slot for *minutes*. `FileLockService`
+ * would normally steal a lock from a *live* owner once it ages past
+ * `liveOwnerStaleThreshold` (to recover a process stalled mid-critical-section).
+ * For a run-cap that is exactly wrong: stealing a live run's slot would silently
+ * admit an (N+1)th concurrent run — reintroducing the saturation this guard
+ * exists to prevent. So slots are configured to *never* steal from a live owner;
+ * only provably-dead owners are reclaimed (immediately, regardless of age).
+ *
+ * ### Concurrency within one process
+ * `acquire()` returns an independent {@link RunSlotAcquisition} handle capturing
+ * its own slot service; it stores *no* shared "held slot" state. Two concurrent
+ * `runResearch` calls on the same singleton service each acquire a distinct slot
+ * safely (see "fresh instance per attempt" above), with no risk of one release
+ * clobbering the other's tracking.
+ */
+
+import * as pathmod from 'node:path';
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+import { FileLockService } from './file-lock-service.ts';
+import { ServiceLifecycle } from '../core/service-registry.ts';
+import { ServiceNames } from '../core/interfaces/service-names.ts';
+import type { IService } from '../core/service-registry.ts';
+import type { IProcessLifecycle } from '../core/interfaces/process-interfaces.ts';
+
+/** Default maximum number of concurrent research runs across all processes. */
+export const DEFAULT_MAX_CONCURRENT_RUNS = 3;
+/**
+ * Default per-acquire wait before giving up when all slots are held by live runs.
+ *
+ * This is deliberately a *queue*, not a fail-fast. The cap exists to stop N runs
+ * saturating the shared browser/embedding pool — but an extra run is a legitimate
+ * request to do work, not an error, so the correct response to contention is to
+ * serialize it behind the in-flight runs rather than to fail it. With a depth-1 run
+ * measured at ~2-3 minutes, 10 minutes absorbs a backlog of roughly three further
+ * runs per slot while still resolving well inside the agent-skill's documented
+ * command timeouts (so a queued run never looks like a hang to the caller).
+ *
+ * Set `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS=0` to restore strict fail-fast.
+ */
+export const DEFAULT_ACQUIRE_TIMEOUT_MS = 600_000;
+/** Poll interval between full slot sweeps while waiting for a slot. */
+const POLL_DELAY_MS = 250;
+
+/**
+ * Thrown when every run slot is held by a live process and the acquire timeout
+ * has elapsed. Distinct from internal/IO errors so callers can treat
+ * "capacity exhausted" (retry later) differently from "the semaphore is broken".
+ */
+export class ResearchRunCapacityError extends Error {
+  readonly slots: number;
+  constructor(slots: number, maxWaitMs: number) {
+    super(
+      `Maximum concurrent research runs (${slots}) reached${maxWaitMs > 0 ? ` and still busy after queueing for ${Math.round(maxWaitMs / 1000)}s` : ''}; ` +
+        `another run is still in progress. This is temporary — retry shortly. ` +
+        `(Raise the cap with PI_RESEARCH_MAX_CONCURRENT_RUNS, or the queue wait with ` +
+        `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS.)`,
+    );
+    this.name = 'ResearchRunCapacityError';
+    this.slots = slots;
+  }
+}
+
+/** A held run slot. Call {@link RunSlotAcquisition.release} exactly once when the run ends. */
+export interface RunSlotAcquisition {
+  /** The 0-based slot index that was acquired (observability). */
+  readonly slotIndex: number;
+  /** Release the slot back to the pool. Safe to call once; subsequent calls are no-ops. */
+  release(): Promise<void>;
+}
+
+/**
+ * Counting semaphore over N slot files coordinated with {@link FileLockService}.
+ */
+export class ResearchRunSemaphore implements IService {
+  readonly name = ServiceNames.RESEARCH_RUN_SEMAPHORE;
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+
+  private readonly slotDir: string;
+  private readonly processLifecycle: IProcessLifecycle | null;
+  private readonly maxSlots: number;
+  private readonly defaultMaxWaitMs: number;
+  /** Rotating start index so contending processes don't all hammer slot 0 first. */
+  private nextStartIndex = 0;
+  private _initialized = false;
+
+  /**
+   * @param slotDir    Directory holding the `research-run-slot-<i>.lock` files.
+   * @param processLifecycle PID+startTime liveness source (PID-reuse-safe reclamation).
+   * @param maxSlots   N — maximum concurrent runs. Defaults from env then {@link DEFAULT_MAX_CONCURRENT_RUNS}.
+   * @param defaultMaxWaitMs Default acquire wait. Defaults from env then {@link DEFAULT_ACQUIRE_TIMEOUT_MS}.
+   */
+  constructor(slotDir: string, processLifecycle: IProcessLifecycle | null, maxSlots?: number, defaultMaxWaitMs?: number) {
+    this.slotDir = slotDir;
+    this.processLifecycle = processLifecycle;
+    this.maxSlots = Math.max(1, Math.floor(
+      maxSlots ?? numberFromEnv('PI_RESEARCH_MAX_CONCURRENT_RUNS') ?? DEFAULT_MAX_CONCURRENT_RUNS,
+    ));
+    this.defaultMaxWaitMs = Math.max(0, Math.floor(
+      defaultMaxWaitMs ?? numberFromEnv('PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS') ?? DEFAULT_ACQUIRE_TIMEOUT_MS,
+    ));
+  }
+
+  /** Number of slots (max concurrent runs). */
+  getMaxSlots(): number {
+    return this.maxSlots;
+  }
+
+  async initialize(): Promise<void> {
+    if (this._initialized) return;
+    this.lifecycle = ServiceLifecycle.INITIALIZING;
+    // Ensure the slot directory exists; slot files are created lazily on acquire.
+    try {
+      await import('node:fs/promises').then((fs) => fs.mkdir(this.slotDir, { recursive: true, mode: 0o700 }));
+    } catch (err) {
+      logger.warn(`[ResearchRunSemaphore] Could not create slot dir ${this.slotDir}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    this._initialized = true;
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+    logger.log(`[ResearchRunSemaphore] Initialized with ${this.maxSlots} run slot(s) (acquire timeout ${this.defaultMaxWaitMs}ms).`);
+  }
+
+  async dispose(): Promise<void> {
+    // Held slots are released by their RunSlotAcquisition handles (the runResearch
+    // finally block). A hard process exit leaves slot files behind, which the next
+    // acquire reclaims via PID+startTime liveness — so dispose has nothing to do.
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+  }
+
+  /**
+   * Try to win one slot NON-BLOCKING. Builds a fresh {@link FileLockService}
+   * configured as a try-take (see class doc: "fresh instance per attempt").
+   * Returns an acquisition on success, or null if the slot is busy / lost a race.
+   */
+  private async trySlot(slotIndex: number): Promise<RunSlotAcquisition | null> {
+    const slot = new FileLockService({
+      lockFilePath: pathmod.join(this.slotDir, `research-run-slot-${slotIndex}.lock`),
+      lockTimeout: 0, // fail immediately on a live-contended slot (try-take)
+      lockRetries: 24, // enough iterations to complete a dead-owner reclaim + reacquire
+      lockRetryDelay: 50,
+      lockStaleThreshold: 15000, // legacy (no-pid) locks: reclaim once 15s old
+      // A live process holding a run slot for the entire (minutes-long) run is the
+      // NORMAL case, not a stall. Never steal from a live owner — doing so would
+      // admit an (N+1)th concurrent run, the exact condition this guard prevents.
+      // Dead owners are reclaimed immediately regardless of age.
+      liveOwnerStaleThreshold: Number.MAX_SAFE_INTEGER,
+      processLifecycle: this.processLifecycle ?? undefined,
+    });
+    try {
+      await slot.initialize(); // resolves own startTime (cached) + startup stale cleanup
+      await slot.acquireLock(); // try-take: wins, reclaims a dead owner, or throws fast
+    } catch {
+      return null; // busy (live owner) or lost a reclaim race — try the next slot
+    }
+    return {
+      slotIndex,
+      release: async () => {
+        try {
+          await slot.releaseLock();
+        } catch (err) {
+          logger.warn(`[ResearchRunSemaphore] Failed to release slot ${slotIndex}: ${err instanceof Error ? err.message : String(err)}`);
+        } finally {
+          // Clear the gauge even if the unlink failed: the slot is no longer held
+          // by THIS process either way, and a gauge that only ever ratchets to 1
+          // reports permanent saturation for the life of the process.
+          metrics.setGauge('research_run_slots_held', 0, { slot: String(slotIndex) });
+        }
+      },
+    };
+  }
+
+  /**
+   * Acquire one run slot, waiting up to `maxWaitMs` (defaults to the constructed
+   * value) when all slots are held by live processes. Resolves with a handle that
+   * must be released when the run ends.
+   *
+   * @param signal aborts the wait. Without it a run started under a long
+   *   `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS` would ignore Ctrl-C until the deadline
+   *   passed, because the wait happens before any orchestrator exists to cancel.
+   * @param onQueued invoked at most once, when the first full sweep finds every
+   *   slot busy and the call is about to wait. Lets a front-end tell the user the
+   *   run is queued rather than hung. Never allowed to break the acquire.
+   * @throws {ResearchRunCapacityError} if no slot frees within `maxWaitMs`.
+   * @throws {Error} named `AbortError` if `signal` aborts while waiting.
+   */
+  async acquire(
+    maxWaitMs: number = this.defaultMaxWaitMs,
+    signal?: AbortSignal,
+    onQueued?: (slots: number, maxWaitMs: number) => void,
+  ): Promise<RunSlotAcquisition> {
+    if (!this._initialized) await this.initialize();
+    if (signal?.aborted) {
+      metrics.increment('research_run_acquire_total', 1, { status: 'aborted' });
+      throwIfAborted(signal);
+    }
+
+    const deadline = Date.now() + Math.max(0, maxWaitMs);
+    let announcedQueued = false;
+    for (;;) {
+      // Round-robin from a rotating start so contending processes don't synchronize on slot 0.
+      const start = this.nextStartIndex;
+      this.nextStartIndex = (this.nextStartIndex + 1) % this.maxSlots;
+      for (let offset = 0; offset < this.maxSlots; offset++) {
+        const i = (start + offset) % this.maxSlots;
+        const got = await this.trySlot(i);
+        if (got) {
+          metrics.setGauge('research_run_slots_held', 1, { slot: String(i) });
+          metrics.increment('research_run_acquire_total', 1, { status: 'acquired' });
+          return got;
+        }
+      }
+      // Every slot is held by a live process.
+      if (Date.now() >= deadline) {
+        metrics.increment('research_run_acquire_total', 1, { status: 'exhausted' });
+        throw new ResearchRunCapacityError(this.maxSlots, maxWaitMs);
+      }
+      // Announce the queue exactly once, on the first sweep that found nothing free.
+      // A throwing front-end callback must never fail the acquire.
+      if (!announcedQueued) {
+        announcedQueued = true;
+        logger.info(`[ResearchRunSemaphore] All ${this.maxSlots} run slot(s) busy — queueing for up to ${maxWaitMs}ms.`);
+        try {
+          onQueued?.(this.maxSlots, maxWaitMs);
+        } catch (err) {
+          logger.debug(`[ResearchRunSemaphore] onQueued callback threw: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      }
+      // Count the aborted outcome too, so acquired + exhausted + aborted accounts for
+      // every acquire attempt. A cancel that vanished from the counter would make the
+      // capacity metrics quietly under-report demand.
+      try {
+        await sleep(POLL_DELAY_MS, signal);
+      } catch (err) {
+        metrics.increment('research_run_acquire_total', 1, { status: 'aborted' });
+        throw err;
+      }
+    }
+  }
+}
+
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw Object.assign(new Error('Research run slot acquisition aborted'), { name: 'AbortError' });
+  }
+}
+
+function numberFromEnv(key: string): number | undefined {
+  const raw = process.env[key];
+  if (raw === undefined || raw === '') return undefined;
+  const n = Number(raw);
+  return Number.isFinite(n) ? n : undefined;
+}
+
+/**
+ * Poll delay that resolves early (rejecting) on abort.
+ *
+ * The timer is deliberately NOT `unref`'d. Waiting for a run slot is foreground work: a process
+ * whose only pending handle is this timer must stay alive until the slot frees, and an unref'd
+ * timer would let Node drain the loop and exit mid-wait (observed as an early exit in the
+ * multi-process harness). The timer is always settled — it either fires or is cleared on abort —
+ * so keeping it referenced cannot outlive the acquire.
+ */
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      if (onAbort) signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    const onAbort = signal
+      ? () => {
+          clearTimeout(timer);
+          reject(Object.assign(new Error('Research run slot acquisition aborted'), { name: 'AbortError' }));
+        }
+      : undefined;
+    if (onAbort) signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}

package/src/infrastructure/service-initialization.ts

@@ -22,6 +22,7 @@ import { StateValidator } from './state/state-validator.ts';
 import { WorkerPoolManager } from './browser/worker-pool-manager.ts';
 import { StatePathConfiguration } from './state/state-path-configuration.ts';
 import { FileLockService } from './file-lock-service.ts';
+import { ResearchRunSemaphore } from './research-run-semaphore.ts';
 import { StateBackupManager } from './state/state-backup-manager.ts';
 import { logger } from '../logger.ts';
 
@@ -73,6 +74,25 @@ export function registerInfrastructureServices(container: ServiceContainer = get
     container
   );
 
+  // Register Research Run Semaphore (Phase 1 cross-process run-cap).
+  // Depends on StatePathConfiguration (slot dir) and ProcessLifecycle (PID-reuse-safe
+  // reclamation). Lazy: only built on first research run. Acquired at runResearch()
+  // entry; released in its finally. A bug here fails OPEN (research proceeds) so it
+  // can never break research; genuine capacity exhaustion fails fast with a clear error.
+  registerService(
+    ServiceNames.RESEARCH_RUN_SEMAPHORE,
+    async () => {
+      const pathConfig = await getService<StatePathConfiguration>(ServiceNames.STATE_PATH_CONFIGURATION, undefined, container);
+      const processLifecycle = await getService<ProcessLifecycleService>(ServiceNames.PROCESS_LIFECYCLE, undefined, container);
+      return new ResearchRunSemaphore(pathConfig.getStateDir(), processLifecycle);
+    },
+    {
+      allowOverwrite: false,
+      enableLogging: true,
+    },
+    container
+  );
+
   // Register State Backup Manager Service
   // Depends on StatePathConfiguration
   registerService(

package/src/orchestration/deep-research-orchestrator.ts

@@ -555,6 +555,16 @@ export class DeepResearchOrchestrator {
         logger.debug('[DeepOrchestrator] onComplete observer threw:', obsErr);
       }
 
+      // Observability: log the FULL final synthesis body. Every other layer —
+      // researcher prompts, researcher final responses, searches, scrapes — is logged
+      // at DEBUG; the synthesis itself (the actual bytes returned to the caller) was
+      // the one conspicuous exception, which made post-hoc diagnosis of citation /
+      // output issues impossible (a reported "the engine returned placeholder X"
+      // could not be confirmed or refuted from the logs). Mirrors the DEBUG logging
+      // of researcher final responses. Truncation is deliberately NOT applied: the
+      // raw bytes are exactly what's needed to diagnose grounding failures.
+      logger.debug(`[DeepOrchestrator] Final synthesis for ${researchId} (${result.length} chars):\n${result}`);
+
       return result;
 
     } catch (error) {
@@ -589,6 +599,7 @@ export class DeepResearchOrchestrator {
           } catch (obsErr) {
             logger.debug('[DeepOrchestrator] fallback onComplete observer threw:', obsErr);
           }
+          logger.debug(`[DeepOrchestrator] Fallback synthesis for ${researchId} (${result.length} chars):\n${result}`);
           return result;
         }
       } catch (fallbackErr) {

package/src/orchestration/headless-observer.ts

@@ -95,6 +95,10 @@ export class HeadlessObserver implements ResearchObserver {
     this.emit('start', { query, complexity });
   }
 
+  onRunQueued(slots: number, maxWaitMs: number): void {
+    this.emit('run_queued', { slots, maxWaitMs });
+  }
+
   onPlanningStart(attempt: number): void {
     this.emit('planning_start', { attempt });
   }