@lincoln504/pi-research 1.2.0 → 1.3.1

package/src/infrastructure/embedding/embedding-factory.ts

@@ -76,6 +76,13 @@ export async function isEmbeddingLeaderAlive(
 
 let _embeddingInstance: IEmbedder | null = null;
 let _embeddingInitPromise: Promise<IEmbedder> | null = null;
+// Config the in-flight init was started for. The join below must compare against
+// these — the cached-instance path checks model/device before reuse, and the
+// adoption paths refuse mismatched leaders, but a caller joining a promise built
+// for a different config would silently receive (and stamp the cache with) the
+// wrong-model embedder, defeating both guards.
+let _initModel: string | null = null;
+let _initDevice: string | null = null;
 let _cachedModel: string | null = null;
 let _cachedDevice: string | null = null;
 
@@ -130,8 +137,18 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
 
   // An internal retry (_attempt > 0) is re-entering from INSIDE the promise stored
   // here, so handing that promise back would make it await itself. Only a fresh
-  // external call may join an in-flight init.
-  if (_embeddingInitPromise && _attempt === 0) return _embeddingInitPromise;
+  // external call may join an in-flight init — and only one started for the SAME
+  // config: a mismatched joiner would get the wrong-model embedder (same-dimension
+  // models cross-embed silently). On mismatch, wait the in-flight init out (never
+  // race a second election in this process), then re-resolve for this config — the
+  // cached-instance check above disposes the mismatched instance.
+  if (_embeddingInitPromise && _attempt === 0) {
+    if (_initModel === cfg.EMBEDDING_MODEL && _initDevice === cfg.EMBEDDING_DEVICE) {
+      return _embeddingInitPromise;
+    }
+    await _embeddingInitPromise.catch(() => {});
+    return getEmbedder(cfg, _attempt);
+  }
 
   let p: Promise<IEmbedder>;
 
@@ -222,7 +239,12 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
         if (!(await isEmbeddingLeaderAlive(info, processLifecycle))) {
           // Candidate process died without cleaning up (or its PID was recycled by
           // an unrelated process — the start-time pairing catches that too).
-          await stateManager.clearEmbeddingServer().catch((err) => logger.debug('Swallowed clear embedding server error:', err));
+          // serverId-scoped: several waiters typically detect the death in the same
+          // poll tick; the first one's clear+claim can commit before this one's clear
+          // runs, and an unconditional delete here would deregister that NEW claimant
+          // (whose Phase-3 port publish is CAS-guarded and silently no-ops) — leaving
+          // two concurrent model inits and an invisible leader.
+          await stateManager.clearEmbeddingServer({ serverId: info.serverId }).catch((err) => logger.debug('Swallowed clear embedding server error:', err));
           break;
         }
         // Re-check model identity every poll: the leader may have changed while
@@ -246,7 +268,10 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
           const secondCheck = await isPortListening(info.port);
           if (!secondCheck) {
             logger.warn(`[EmbeddingFactory] Registered port ${info.port} is unreachable after two checks — clearing stale state`);
-            await stateManager.clearEmbeddingServer().catch((err) => logger.debug('Swallowed clear embedding server error:', err));
+            // serverId-scoped: this snapshot is up to ~2.5s old (two probes + sleep);
+            // the entry may already belong to a new claimant. See the dead-leader
+            // clear above for the failure this prevents.
+            await stateManager.clearEmbeddingServer({ serverId: info.serverId }).catch((err) => logger.debug('Swallowed clear embedding server error:', err));
             break;
           }
         }
@@ -282,8 +307,10 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
     try {
       port = await server.startServer();
     } catch (err) {
-      // Init failed — clear our placeholder so others can try
-      await stateManager.clearEmbeddingServer().catch((err) => logger.debug('Swallowed clear embedding server error:', err));
+      // Init failed — clear our placeholder so others can try. serverId-scoped so a
+      // concurrent steal-and-reclaim (a waiter that judged us dead and claimed) is
+      // never deregistered by our own failure path.
+      await stateManager.clearEmbeddingServer({ serverId }).catch((err) => logger.debug('Swallowed clear embedding server error:', err));
       throw err;
     }
 
@@ -307,6 +334,8 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
 
   p = init();
   _embeddingInitPromise = p;
+  _initModel = cfg.EMBEDDING_MODEL;
+  _initDevice = cfg.EMBEDDING_DEVICE;
   // init() already assigns _embeddingInstance before it resolves, so do NOT
   // re-assign it here. If clearEmbeddingInstance() ran (and disposed the server)
   // between init resolving and this microtask, re-assigning would resurrect the

package/src/infrastructure/embedding/embedding-server.ts

@@ -16,6 +16,7 @@ import { Utf8Body } from '../../utils/http-body.ts';
 import type { IEmbedder } from '../../core/interfaces/knowledge-interfaces.ts';
 import type { IStateManager } from '../../core/interfaces/state-manager-interfaces.ts';
 import type { Embedder } from '../../knowledge/embedder.ts';
+import { markWebGpuFallback } from '../../knowledge/embedder-utils.ts';
 
 /**
  * Per-process shared secret every embed request must present.
@@ -49,6 +50,24 @@ function isAuthorized(presented: string | string[] | undefined): boolean {
   return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
 }
 
+/**
+ * Refuse to serialize a vector containing NaN/±Infinity. JSON.stringify emits
+ * `null` for non-finite numbers, and the client's `new Float32Array(resp.data)`
+ * coerces null back to 0 — a silently part-zeroed vector that passes every
+ * downstream width/finite gate (0 IS finite) and lands in the ANN index. Failing
+ * the request instead surfaces it as an ordinary embed failure on the client.
+ */
+function assertFiniteVector(vec: ArrayLike<number>, vectorIndex?: number): void {
+  for (let i = 0; i < vec.length; i++) {
+    if (!Number.isFinite(vec[i]!)) {
+      const where = vectorIndex === undefined ? `position ${i}` : `vector ${vectorIndex}, position ${i}`;
+      throw new Error(
+        `[EmbeddingServer] embedder produced a non-finite value (${vec[i]}) at ${where} — refusing to serialize (JSON would launder it to null/0)`,
+      );
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------
 // SerialQueue — ensures embed/embedMany never run concurrently on the GPU
 // ---------------------------------------------------------------------------
@@ -274,6 +293,15 @@ export class EmbeddingServer implements IEmbedder {
       'Stepping down as leader so a fresh process can re-elect with a clean GPU context. ' +
       err.message,
     );
+    // INVARIANT: after a poisoned (permanently hung) inference, THIS process must
+    // never touch WebGPU again. The wedged native call still holds the same-PID
+    // re-entrant GPU lock, so if this process wins the next election and
+    // re-initializes WebGPU it would run a second inference beside the stuck one —
+    // the concurrent-GPU-inference segfault class the SerialQueue exists to
+    // prevent. Latch the process-wide fallback so any embedder built in this
+    // process from here on comes up on CPU; only a FRESH process may re-elect
+    // with a clean GPU context.
+    markWebGpuFallback();
     // Step down WITHOUT disposing the embedder: pipeline.dispose() would join the
     // stuck native thread (itself hanging) or touch a device-lost context. Clearing
     // the state registration + closing the HTTP server is enough for re-election;
@@ -339,14 +367,14 @@ export class EmbeddingServer implements IEmbedder {
       this.leadershipTimer = null;
     }
 
-    // Clear our registration if we still own it
+    // Clear our registration if we still own it. The serverId CAS runs inside the
+    // state lock, so a successor that claimed the slot between our last check and
+    // this shutdown can never be deregistered by us (the old read-then-clear had
+    // exactly that window).
     try {
-      const serverInfo = await this.stateManager.getEmbeddingServer();
-      if (serverInfo?.serverId === this.serverId) {
-        await this.stateManager.clearEmbeddingServer().catch((err) => {
-          logger.warn('[EmbeddingServer] Failed to clear embedding server state:', err);
-        });
-      }
+      await this.stateManager.clearEmbeddingServer({ serverId: this.serverId }).catch((err) => {
+        logger.warn('[EmbeddingServer] Failed to clear embedding server state:', err);
+      });
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
       // State manager is torn down before the embedding server during process exit —
@@ -552,6 +580,9 @@ export class EmbeddingServer implements IEmbedder {
               // message drives the client to reconnect to the new leader.
               this.assertServing();
               const result = await this.queue.enqueue(() => this.embedder.embed(data.text as string));
+              // Scan BEFORE serialization: a device-lost mid-inference yields NaN
+              // entries that JSON would launder into null → 0 on the client.
+              assertFiniteVector(result);
               res.writeHead(200, { 'Content-Type': 'application/json' });
               res.end(JSON.stringify({ data: Array.from(result) }));
               break;
@@ -559,6 +590,8 @@ export class EmbeddingServer implements IEmbedder {
             case '/embedMany': {
               this.assertServing();
               const results = await this.queue.enqueue(() => this.embedder.embedMany(data.texts as string[]));
+              // Same non-finite scan as /embed, per vector (see assertFiniteVector).
+              for (let i = 0; i < results.length; i++) assertFiniteVector(results[i]!, i);
               res.writeHead(200, { 'Content-Type': 'application/json' });
               res.end(JSON.stringify({ results: results.map(r => Array.from(r)) }));
               break;

package/src/infrastructure/file-lock-service.ts

@@ -38,6 +38,16 @@ const CLEANUP_DRAIN_TIMEOUT_MS = 5_000;
  */
 const TRASH_SWEEP_AGE_MS = 10 * 60_000;
 
+/**
+ * Floor for the held-lock mtime heartbeat interval (see _startHeartbeat), so a
+ * small liveOwnerStaleThreshold cannot turn maintenance into a utimes storm.
+ */
+const HEARTBEAT_MIN_INTERVAL_MS = 2_000;
+
+/** setTimeout/setInterval clamp delays above 2^31-1 to 1ms — treat anything
+ *  bigger as "no heartbeat" rather than a busy-loop. */
+const MAX_TIMER_DELAY_MS = 0x7fffffff;
+
 /**
  * Lock configuration options
  */
@@ -101,9 +111,17 @@ export class FileLockService implements IService {
   // Set by cleanup(). A retired instance no longer manages the lock file, so
   // acquiring through it would hand out exclusivity it cannot honour.
   private _retired = false;
+  // Refreshes the held lock file's mtime while a critical section runs (see
+  // _startHeartbeat). Null whenever no lock is held.
+  private heartbeatTimer: NodeJS.Timeout | null = null;
+  // Stable short lock identity for metrics labels — without it every instance
+  // (state lock, knowledge init, browser init, run slots) shares one gauge and
+  // any release zeroes it globally.
+  private readonly lockName: string;
 
   constructor(options: FileLockOptions) {
     this.lockFilePath = options.lockFilePath;
+    this.lockName = pathmod.basename(options.lockFilePath);
     this.lockTimeout = options.lockTimeout ?? 20000;
     this.lockRetries = options.lockRetries ?? 200;
     this.lockRetryDelay = options.lockRetryDelay ?? 100;
@@ -254,6 +272,48 @@ export class FileLockService implements IService {
     if (resolve) resolve();
   }
 
+  /**
+   * While the lock is HELD, refresh its mtime on an interval well inside
+   * liveOwnerStaleThreshold. Peers judge a live owner purely by lock-file age
+   * (_shouldReclaim), and nothing else touches the file during a critical
+   * section — so without a heartbeat a legitimately long one (e.g. a multi-minute
+   * knowledge-store model-change migration under the 240s init-lock threshold)
+   * ages past the limit and is stolen from a LIVE owner, admitting a concurrent
+   * writer. With the heartbeat, a live owner is only stolen after genuinely
+   * wedging: no refresh for the full threshold.
+   *
+   * unref() is CORRECT here, unlike a timer that IS the operation: the heartbeat
+   * is maintenance on behalf of a critical section that keeps the process alive
+   * by itself, and the process must never be kept alive just to freshen a lock.
+   *
+   * The refresh goes through the held FileHandle (futimes), not the path, so if
+   * the lock changed hands (renamed/unlinked by a reclaiming peer) we can only
+   * ever touch OUR old inode — never freshen a peer's lock by mistake.
+   */
+  private _startHeartbeat(): void {
+    this._stopHeartbeat();
+    const interval = Math.max(HEARTBEAT_MIN_INTERVAL_MS, Math.ceil(this.liveOwnerStaleThreshold / 4));
+    // An effectively-infinite threshold (the run-semaphore's never-steal-from-a-
+    // live-owner opt-out) needs no heartbeat — and would overflow the 32-bit
+    // timer delay, which Node clamps to 1ms (a utimes busy-loop). Skip entirely;
+    // the opt-out's semantics are unchanged.
+    if (interval > MAX_TIMER_DELAY_MS) return;
+    this.heartbeatTimer = setInterval(() => {
+      const handle = this.lockHandle;
+      if (handle === null) return;
+      const now = new Date();
+      handle.utimes(now, now).catch(() => { /* best-effort maintenance */ });
+    }, interval);
+    this.heartbeatTimer.unref?.();
+  }
+
+  private _stopHeartbeat(): void {
+    if (this.heartbeatTimer !== null) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
+
   /**
    * Wait, bounded, for an in-flight critical section to finish.
    *
@@ -448,6 +508,9 @@ export class FileLockService implements IService {
           // Fully acquired — commit to instance state
           this.lockHandle = handle;
           this.lockCount = 1;
+          // Keep the held lock's mtime fresh so a long critical section is not
+          // mistaken for a wedged owner and stolen (see _startHeartbeat).
+          this._startHeartbeat();
 
           // Track in ALS for nested calls
           if (heldLocks) {
@@ -457,7 +520,7 @@ export class FileLockService implements IService {
           const duration = Date.now() - startTime;
           metrics.observe('state_lock_acquire_duration_ms', duration);
           metrics.increment('state_lock_acquire_total', 1, { status: 'success' });
-          metrics.setGauge('state_lock_held', 1);
+          metrics.setGauge('state_lock_held', 1, { lock: this.lockName });
           if (contentionCount > 0) {
             metrics.increment('state_lock_contention_total', 1);
             metrics.observe('state_lock_contention_retries', contentionCount);
@@ -596,6 +659,10 @@ export class FileLockService implements IService {
       return;
     }
 
+    // Fully releasing (not a re-entrant pop) — the mtime heartbeat must stop
+    // with the hold it maintains.
+    this._stopHeartbeat();
+
     try {
       // 1. Double check ownership before unlinking
       const content = await fs.readFile(this.lockFilePath, 'utf8').catch(() => '');
@@ -615,7 +682,7 @@ export class FileLockService implements IService {
         heldLocks.delete(`${this.lockFilePath}:${this.lockUuid}`);
       }
 
-      metrics.setGauge('state_lock_held', 0);
+      metrics.setGauge('state_lock_held', 0, { lock: this.lockName });
     } finally {
       // 4. Signal the next caller in the queue that it's their turn.
       this._releaseTurn();
@@ -674,6 +741,9 @@ export class FileLockService implements IService {
     // Retire BEFORE releasing the turn, so a waiter woken below re-checks and
     // fails fast instead of acquiring a lock this instance no longer manages.
     this._retired = true;
+    // The forced-teardown path bypasses releaseLock, so stop the heartbeat here
+    // too — a retired instance must not keep freshening a lock it forfeited.
+    this._stopHeartbeat();
 
     if (this.lockHandle) {
       try {

package/src/infrastructure/knowledge-store-service.ts

@@ -108,10 +108,40 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
       return this._initializationPromise;
     }
 
+    // A different-cwd (or post-dispose-gate) call while an init is still in flight
+    // must NOT start a second closure: the two would race assignments to
+    // _embedder/_store/_writerQueue (the loser's components leak un-disposed), and
+    // the first to settle would null the second's promise marker — letting a
+    // concurrent dispose() skip its wait-for-init guard and be resurrected when the
+    // straggler flips lifecycle back to INITIALIZED. Settle the in-flight init
+    // first, then fall through to the normal re-init path.
+    while (this._initializationPromise) {
+      await this._initializationPromise.catch(() => { /* its own catch reset state */ });
+      // A newer init may have started while we awaited (or while dispose() below
+      // yielded); if it covers our cwd, join it — the loop re-checks either way.
+      if (this._initializationPromise && this._cwd === newCwd) {
+        return this._initializationPromise;
+      }
+      if (this.lifecycle === ServiceLifecycle.INITIALIZED || this.lifecycle === ServiceLifecycle.DISABLED) {
+        if (this._cwd !== newCwd) {
+          logger.log(`[KnowledgeStoreService] CWD changed from ${this._cwd} to ${newCwd}. Re-initializing store...`);
+          await this.dispose();
+        } else {
+          // The settled init already covers this cwd.
+          return;
+        }
+      }
+    }
+
     this.lifecycle = ServiceLifecycle.INITIALIZING;
     logger.debug('[KnowledgeStoreService] Initializing...');
 
-    this._initializationPromise = (async () => {
+    // `let ... = null` (not const): if the closure throws BEFORE its first await,
+    // the finally runs synchronously during the IIFE call — a const self-reference
+    // would be a TDZ crash. The null guard in the finally covers that window (the
+    // marker was never published, so there is nothing to clear).
+    let initPromise: Promise<void> | null = null;
+    initPromise = (async () => {
       try {
         // Create the knowledge store components.
         // Prefer an explicit ctx.cwd; otherwise keep the cwd already resolved on this
@@ -228,11 +258,18 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         this.lifecycle = ServiceLifecycle.UNINITIALIZED;
         throw err;
       } finally {
-        this._initializationPromise = null;
+        // Identity-guarded (same pattern as embedding-factory's clearIfCurrent): only
+        // the promise that owns the marker may null it, so a settling straggler can
+        // never disarm a newer init's marker — which dispose() relies on to wait out
+        // in-flight work.
+        if (this._initializationPromise === initPromise) {
+          this._initializationPromise = null;
+        }
       }
     })();
+    this._initializationPromise = initPromise;
 
-    return this._initializationPromise;
+    return initPromise;
   }
 
   async dispose(): Promise<void> {
@@ -267,6 +304,14 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
 
       if (this._embedder) {
         await this._embedder.dispose?.();
+        // The embedding factory's module-level cache still points at the instance
+        // just disposed — its fast path has no liveness check, so a later re-init
+        // (cwd/mode re-scope) would be handed the dead instance and burn every
+        // warm-up/init retry on it. clearEmbeddingInstance() re-disposes
+        // (idempotent) and nulls the cache; when this process was the leader, its
+        // shutdown deregisters via the serverId CAS, and a client instance's
+        // dispose is a no-op — a FOREIGN leader's registration is never touched.
+        await clearEmbeddingInstance();
       }
 
       if (this._initLock) {

package/src/infrastructure/process-lifecycle-service.ts

@@ -55,6 +55,17 @@ export class ProcessLifecycleService implements IProcessLifecycle {
 
   private cachedStartTime: number | null = null;
 
+  // Short-TTL memo for SUBPROCESS-backed start-time lookups (macOS `ps`, Windows
+  // powershell — Linux reads /proc and stays uncached). A queued run-slot waiter
+  // sweeps every 250ms and builds a fresh FileLockService per attempt, so without
+  // a cross-attempt memo each live owner costs up to two `ps` spawns per sweep —
+  // ~48 subprocess spawns/second sustained on macOS while queued. A live PID's
+  // start time never changes; the only staleness is a PID recycled within the
+  // TTL reading as its dead predecessor, which merely delays a stale-owner steal
+  // by up to the TTL (well under every staleness threshold in use).
+  private static readonly START_TIME_TTL_MS = 5000;
+  private startTimeMemo = new Map<number, { value: number | null; at: number }>();
+
   async initialize(): Promise<void> {
     this.lifecycle = ServiceLifecycle.INITIALIZED;
   }
@@ -87,6 +98,18 @@ export class ProcessLifecycleService implements IProcessLifecycle {
       }
     }
 
+    const memoized = this.startTimeMemo.get(pid);
+    if (memoized && Date.now() - memoized.at < ProcessLifecycleService.START_TIME_TTL_MS) {
+      return memoized.value;
+    }
+    const memoize = (value: number | null): number | null => {
+      // Bound the map: sweeps interrogate a handful of owners; anything beyond a
+      // few hundred distinct PIDs inside one TTL window means leaked entries.
+      if (this.startTimeMemo.size > 256) this.startTimeMemo.clear();
+      this.startTimeMemo.set(pid, { value, at: Date.now() });
+      return value;
+    };
+
     // Windows: use powershell to get process creation time
     if (process.platform === 'win32') {
       try {
@@ -99,11 +122,11 @@ export class ProcessLifecycleService implements IProcessLifecycle {
           );
           child.unref?.();
         });
-        if (!output || !output.trim()) return null;
+        if (!output || !output.trim()) return memoize(null);
         const startTime = parseInt(output.trim(), 10);
-        return isNaN(startTime) ? null : startTime;
+        return memoize(isNaN(startTime) ? null : startTime);
       } catch {
-        return null;
+        return memoize(null);
       }
     }
 
@@ -121,8 +144,8 @@ export class ProcessLifecycleService implements IProcessLifecycle {
     // So: try `etimes`, and fall back to parsing POSIX `etime` when it yields
     // nothing. `etime` is mandated by POSIX, so the fallback works everywhere.
     const elapsedSec = await this.readElapsedSeconds(pid);
-    if (elapsedSec === null) return null;
-    return Math.floor(Date.now() / 1000) - elapsedSec;
+    if (elapsedSec === null) return memoize(null);
+    return memoize(Math.floor(Date.now() / 1000) - elapsedSec);
   }
 
   /** Elapsed seconds for `pid` via ps, or null. Tries `etimes`, then POSIX `etime`. */

package/src/infrastructure/research-run-semaphore.ts

@@ -195,7 +195,18 @@ export class ResearchRunSemaphore implements IService {
     try {
       await slot.initialize(); // resolves own startTime (cached) + startup stale cleanup
       await slot.acquireLock(); // try-take: wins, reclaims a dead owner, or throws fast
-    } catch {
+    } catch (err) {
+      // Contention and breakage are different outcomes and must not be conflated.
+      // FileLockService synthesizes its own Errors (no errno code) for the two
+      // contention shapes a try-take produces — a live-contended slot ("timeout
+      // after 0ms", cause EEXIST) and a lost reclaim race ("after N retries") —
+      // and those genuinely mean "slot busy". A real I/O failure (ENOENT/EACCES/
+      // ENOSPC: the slot file is uncreatable; both mkdirs are warn-only) instead
+      // propagates the raw errno error. Swallowing that as "busy" would poll out
+      // the whole acquire window and reject with a FALSE ResearchRunCapacityError,
+      // making the documented fail-open contract (research proceeds when the
+      // semaphore is broken) unreachable. Rethrow so acquire() can tell them apart.
+      if (typeof (err as NodeJS.ErrnoException | null)?.code === 'string') throw err;
       return null; // busy (live owner) or lost a reclaim race — try the next slot
     }
     return {
@@ -246,16 +257,39 @@ export class ResearchRunSemaphore implements IService {
       // Round-robin from a rotating start so contending processes don't synchronize on slot 0.
       const start = this.nextStartIndex;
       this.nextStartIndex = (this.nextStartIndex + 1) % this.maxSlots;
+      // Slot BREAKAGE (I/O error) is tracked separately from contention within
+      // this sweep — a broken slot must never be reported as a busy one.
+      let brokenSlots = 0;
+      let firstBreakage: unknown = null;
       for (let offset = 0; offset < this.maxSlots; offset++) {
         const i = (start + offset) % this.maxSlots;
-        const got = await this.trySlot(i);
+        let got: RunSlotAcquisition | null;
+        try {
+          got = await this.trySlot(i);
+        } catch (err) {
+          // This slot is broken, not busy. Keep sweeping — another slot may still
+          // be acquirable (mixed case: slot 0 uncreatable, slot 1 free).
+          brokenSlots++;
+          firstBreakage ??= err;
+          logger.debug(`[ResearchRunSemaphore] Slot ${i} errored (broken, not busy): ${err instanceof Error ? err.message : String(err)}`);
+          continue;
+        }
         if (got) {
           metrics.setGauge('research_run_slots_held', 1, { slot: String(i) });
           metrics.increment('research_run_acquire_total', 1, { status: 'acquired' });
           return got;
         }
       }
-      // Every slot is held by a live process.
+      // EVERY slot errored: the semaphore itself is broken, not at capacity.
+      // Waiting cannot help — nothing is held that could free up — and a
+      // ResearchRunCapacityError here would misclassify breakage as retryable
+      // contention while the caller's fail-open path (research proceeds without
+      // the cap) stays unreachable. Rethrow the I/O error instead.
+      if (brokenSlots === this.maxSlots) {
+        metrics.increment('research_run_acquire_total', 1, { status: 'error' });
+        throw firstBreakage;
+      }
+      // Every remaining slot is held by a live process.
       if (Date.now() >= deadline) {
         metrics.increment('research_run_acquire_total', 1, { status: 'exhausted' });
         throw new ResearchRunCapacityError(this.maxSlots, maxWaitMs);

package/src/infrastructure/state/state-browser-api.ts

@@ -50,12 +50,15 @@ export class StateBrowserApi {
   /**
    * Clear the browser server information
    * @param updateState Function to update state atomically
+   * @param expected Optional expected-owner identity forwarded to the CAS check
+   *   in StateBrowserManager.clearBrowserServer.
    */
   async clearBrowserServer(
-    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>
+    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>,
+    expected?: { pid?: number; schedulerId?: string }
   ): Promise<void> {
     await updateState((state) => {
-      return this.browserManager.clearBrowserServer(state);
+      return this.browserManager.clearBrowserServer(state, expected);
     });
   }
 

package/src/infrastructure/state/state-browser-manager.ts

@@ -47,9 +47,23 @@ export class StateBrowserManager implements IService {
   /**
    * Clear the browser server information
    * @param state The current state
+   * @param expected Optional identity the caller believes is currently registered.
+   *   When supplied, this is a compare-and-delete: the entry is only removed if it
+   *   still matches (mirrors StateManager.clearEmbeddingServer). browserServer is
+   *   shared cross-process state, and an unconditional delete lets a caller who
+   *   decided the *previous* leader was dead deregister a DIFFERENT process's
+   *   fresh, live registration that claimed the slot in the meantime — that new
+   *   leader then runs undiscoverable (state shows no browserServer), so the next
+   *   getScheduler() caller elects a second, redundant leader. A mismatch means
+   *   someone else already claimed the slot, and the clear must no-op.
    * @returns Updated state without browser server info
    */
-  clearBrowserServer(state: SingletonState): SingletonState {
+  clearBrowserServer(state: SingletonState, expected?: { pid?: number; schedulerId?: string }): SingletonState {
+    const entry = state.browserServer;
+    if (entry && expected !== undefined) {
+      if (expected.schedulerId !== undefined && entry.schedulerId !== expected.schedulerId) return state;
+      if (expected.pid !== undefined && entry.pid !== expected.pid) return state;
+    }
     delete state.browserServer;
     return state;
   }

package/src/infrastructure/state/state-manager-service.ts

@@ -188,8 +188,8 @@ export class StateManagerService implements IStateManager {
   /**
    * Clear the browser server information
    */
-  async clearBrowserServer(): Promise<void> {
-    return this.getStateManager().clearBrowserServer();
+  async clearBrowserServer(expected?: { pid?: number; schedulerId?: string }): Promise<void> {
+    return this.getStateManager().clearBrowserServer(expected);
   }
 
   /**
@@ -202,8 +202,8 @@ export class StateManagerService implements IStateManager {
   /**
    * Clear the embedding server information
    */
-  async clearEmbeddingServer(): Promise<void> {
-    return this.getStateManager().clearEmbeddingServer();
+  async clearEmbeddingServer(expected?: { pid?: number; serverId?: string }): Promise<void> {
+    return this.getStateManager().clearEmbeddingServer(expected);
   }
 
   /**

package/src/infrastructure/state/state-manager.ts

@@ -446,8 +446,8 @@ export class StateManager {
     );
   }
 
-  public async clearBrowserServer(): Promise<void> {
-    await this.browserApi.clearBrowserServer(this.updateState.bind(this));
+  public async clearBrowserServer(expected?: { pid?: number; schedulerId?: string }): Promise<void> {
+    await this.browserApi.clearBrowserServer(this.updateState.bind(this), expected);
   }
 
   // ==================== Embedding Server API ====================
@@ -457,8 +457,20 @@ export class StateManager {
     return state.embeddingServer ?? null;
   }
 
-  public async clearEmbeddingServer(): Promise<void> {
+  public async clearEmbeddingServer(expected?: { pid?: number; serverId?: string }): Promise<void> {
     await this.updateState((state) => {
+      // Compare-and-delete: the registration is shared cross-process state, and an
+      // unconditional delete lets a follower's shutdown (or a waiter acting on a
+      // stale snapshot) deregister a DIFFERENT process's live leader — which then
+      // never re-registers (its leadership check treats an absent entry as benign),
+      // so a fresh caller elects a second GPU model instance alongside the invisible
+      // one. Callers pass the identity they believe they are clearing; a mismatch
+      // means someone else already claimed the slot, and the clear must no-op.
+      const entry = state.embeddingServer;
+      if (entry && expected !== undefined) {
+        if (expected.serverId !== undefined && entry.serverId !== expected.serverId) return state;
+        if (expected.pid !== undefined && entry.pid !== expected.pid) return state;
+      }
       delete state.embeddingServer;
       return state;
     });