@lincoln504/pi-research 1.1.2 → 1.3.0

package/src/infrastructure/file-lock-service.ts

@@ -23,6 +23,31 @@ import type { IProcessLifecycle } from '../core/interfaces/process-interfaces.ts
  */
 const lockContext = new AsyncLocalStorage<Set<string>>();
 
+/**
+ * How long cleanup() waits for an in-flight critical section to release normally
+ * before forcing the teardown. Long enough for an ordinary state write (temp file
+ * + fsync + rename) to finish, short enough that shutdown still makes progress
+ * when the holder is wedged.
+ */
+const CLEANUP_DRAIN_TIMEOUT_MS = 5_000;
+
+/**
+ * Age past which an orphaned `<lock>.trash.<hex>` copy is swept at startup. An
+ * in-flight trash-and-verify holds its copy for milliseconds; anything this old
+ * was leaked by a crash mid-dance.
+ */
+const TRASH_SWEEP_AGE_MS = 10 * 60_000;
+
+/**
+ * Floor for the held-lock mtime heartbeat interval (see _startHeartbeat), so a
+ * small liveOwnerStaleThreshold cannot turn maintenance into a utimes storm.
+ */
+const HEARTBEAT_MIN_INTERVAL_MS = 2_000;
+
+/** setTimeout/setInterval clamp delays above 2^31-1 to 1ms — treat anything
+ *  bigger as "no heartbeat" rather than a busy-loop. */
+const MAX_TIMER_DELAY_MS = 0x7fffffff;
+
 /**
  * Lock configuration options
  */
@@ -83,9 +108,20 @@ export class FileLockService implements IService {
   // PID reuse. Null when unavailable (no processLifecycle, or lookup failed) —
   // then lock content omits startTime and readers treat us as a legacy owner.
   private ownStartTime: number | null = null;
+  // Set by cleanup(). A retired instance no longer manages the lock file, so
+  // acquiring through it would hand out exclusivity it cannot honour.
+  private _retired = false;
+  // Refreshes the held lock file's mtime while a critical section runs (see
+  // _startHeartbeat). Null whenever no lock is held.
+  private heartbeatTimer: NodeJS.Timeout | null = null;
+  // Stable short lock identity for metrics labels — without it every instance
+  // (state lock, knowledge init, browser init, run slots) shares one gauge and
+  // any release zeroes it globally.
+  private readonly lockName: string;
 
   constructor(options: FileLockOptions) {
     this.lockFilePath = options.lockFilePath;
+    this.lockName = pathmod.basename(options.lockFilePath);
     this.lockTimeout = options.lockTimeout ?? 20000;
     this.lockRetries = options.lockRetries ?? 200;
     this.lockRetryDelay = options.lockRetryDelay ?? 100;
@@ -175,11 +211,153 @@ export class FileLockService implements IService {
     this.lifecycle = ServiceLifecycle.DISPOSED;
   }
 
+  /**
+   * Put a lock file back after a trash-and-verify decided not to reclaim it.
+   *
+   * The old form linked it back inside a swallowing catch and then unlinked the
+   * trash copy unconditionally — so any link failure other than "a peer already
+   * re-created it" DESTROYED the lock this branch exists to preserve. Distinguish
+   * the two: if the path is occupied again the peer's lock wins and ours is dropped;
+   * otherwise move ours back rather than deleting it.
+   */
+  private async _restoreLockFile(trashPath: string): Promise<void> {
+    try {
+      await fs.link(trashPath, this.lockFilePath);
+      await fs.unlink(trashPath).catch(() => { /* link succeeded; copy is redundant */ });
+      return;
+    } catch { /* fall through to the two cases below */ }
+
+    const peerHoldsIt = await fs.access(this.lockFilePath).then(() => true).catch(() => false);
+    if (peerHoldsIt) {
+      // Someone re-created the lock while we were checking. Theirs is authoritative.
+      await fs.unlink(trashPath).catch(() => { /* best effort */ });
+      return;
+    }
+    // Nothing there and link failed (EPERM, filesystems without hardlinks,
+    // Windows link limits): re-create the path EXCLUSIVELY rather than rename it
+    // back — rename silently REPLACES a lock a peer creates between the probe
+    // above and now, clobbering a live owner. 'wx' loses that race loudly, and
+    // then the peer's lock is authoritative.
+    try {
+      const content = await fs.readFile(trashPath, 'utf-8');
+      const handle = await fs.open(this.lockFilePath, 'wx', 0o600);
+      try {
+        await handle.write(content);
+        await handle.sync();
+      } finally {
+        await handle.close().catch(() => { /* content is on disk */ });
+      }
+      await fs.unlink(trashPath).catch(() => { /* restored; copy is redundant */ });
+    } catch {
+      // EEXIST (a peer took the path) — ours is redundant; anything else, leave
+      // the trash copy for the startup sweep. Never delete what we could not
+      // put back.
+      const occupied = await fs.access(this.lockFilePath).then(() => true).catch(() => false);
+      if (occupied) await fs.unlink(trashPath).catch(() => { /* best effort */ });
+    }
+  }
+
+  /**
+   * Hand our turn back to the next caller in the FIFO queue.
+   *
+   * Every path that stops holding the lock must run this, including the ones that
+   * find the handle already gone. `acquireLock` parks on `await previous` with no
+   * timeout of its own — the lockTimeout only starts counting once the turn has
+   * been granted — so a turn that is never resolved is not a slow acquire, it is a
+   * permanent one.
+   */
+  private _releaseTurn(): void {
+    const resolve = this.resolveTurn;
+    this.resolveTurn = null;
+    if (resolve) resolve();
+  }
+
+  /**
+   * While the lock is HELD, refresh its mtime on an interval well inside
+   * liveOwnerStaleThreshold. Peers judge a live owner purely by lock-file age
+   * (_shouldReclaim), and nothing else touches the file during a critical
+   * section — so without a heartbeat a legitimately long one (e.g. a multi-minute
+   * knowledge-store model-change migration under the 240s init-lock threshold)
+   * ages past the limit and is stolen from a LIVE owner, admitting a concurrent
+   * writer. With the heartbeat, a live owner is only stolen after genuinely
+   * wedging: no refresh for the full threshold.
+   *
+   * unref() is CORRECT here, unlike a timer that IS the operation: the heartbeat
+   * is maintenance on behalf of a critical section that keeps the process alive
+   * by itself, and the process must never be kept alive just to freshen a lock.
+   *
+   * The refresh goes through the held FileHandle (futimes), not the path, so if
+   * the lock changed hands (renamed/unlinked by a reclaiming peer) we can only
+   * ever touch OUR old inode — never freshen a peer's lock by mistake.
+   */
+  private _startHeartbeat(): void {
+    this._stopHeartbeat();
+    const interval = Math.max(HEARTBEAT_MIN_INTERVAL_MS, Math.ceil(this.liveOwnerStaleThreshold / 4));
+    // An effectively-infinite threshold (the run-semaphore's never-steal-from-a-
+    // live-owner opt-out) needs no heartbeat — and would overflow the 32-bit
+    // timer delay, which Node clamps to 1ms (a utimes busy-loop). Skip entirely;
+    // the opt-out's semantics are unchanged.
+    if (interval > MAX_TIMER_DELAY_MS) return;
+    this.heartbeatTimer = setInterval(() => {
+      const handle = this.lockHandle;
+      if (handle === null) return;
+      const now = new Date();
+      handle.utimes(now, now).catch(() => { /* best-effort maintenance */ });
+    }, interval);
+    this.heartbeatTimer.unref?.();
+  }
+
+  private _stopHeartbeat(): void {
+    if (this.heartbeatTimer !== null) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
+
+  /**
+   * Wait, bounded, for an in-flight critical section to finish.
+   *
+   * Yanking a held lock is not a clean teardown: unlinking the file while the
+   * holder is still mid read-modify-write drops cross-process exclusion, and a peer
+   * that acquires in that window loses the holder's update — precisely the hazard
+   * the lock exists to prevent. Since every caller of {@link cleanup} is a shutdown
+   * path, waiting briefly for a normal release is both safe and correct.
+   *
+   * Bounded because shutdown must make progress regardless: if the holder is wedged
+   * (or is itself the caller, which would otherwise deadlock), we give up and force
+   * the teardown, which is the pre-existing behaviour.
+   */
+  private async _drainHeldLock(timeoutMs: number): Promise<boolean> {
+    const deadline = Date.now() + timeoutMs;
+    while (this.lockHandle !== null && Date.now() < deadline) {
+      await this.sleep(Math.min(25, this.lockRetryDelay));
+    }
+    return this.lockHandle === null;
+  }
+
   /**
    * Clean up any stale lock files on initialization.
    * This handles cases where locks weren't released due to crashes.
    */
   private async cleanupStaleLocksOnStartup(): Promise<void> {
+    // Sweep trash copies leaked by an interrupted trash-and-verify (a crash or
+    // I/O failure between the rename and the unlink/restore). Nothing else ever
+    // collects them: the state-dir orphan sweep matches only *.tmp files. The age
+    // gate is generous — an in-flight reclaim's trash copy lives for milliseconds,
+    // and a restore-worthy one is put back on the same code path that made it.
+    try {
+      const dir = pathmod.dirname(this.lockFilePath);
+      const prefix = `${pathmod.basename(this.lockFilePath)}.trash.`;
+      for (const name of await fs.readdir(dir)) {
+        if (!name.startsWith(prefix)) continue;
+        const p = pathmod.join(dir, name);
+        const st = await fs.stat(p).catch(() => null);
+        if (st && Date.now() - st.mtimeMs > TRASH_SWEEP_AGE_MS) {
+          await fs.unlink(p).catch(() => { /* best effort */ });
+        }
+      }
+    } catch { /* best-effort hygiene; never blocks startup */ }
+
     try {
       const rawContent = await fs.readFile(this.lockFilePath, 'utf-8').catch(() => null);
       if (rawContent === null) return; // No lock file — nothing to do
@@ -201,19 +379,25 @@ export class FileLockService implements IService {
         // the UUID matches what we decided on — same guard as the contended path.
         const expectedUuid = parsed?.uuid ?? '';
         const trashPath = `${this.lockFilePath}.trash.${crypto.randomBytes(8).toString('hex')}`;
+        let renamed = false;
         try {
           await fs.rename(this.lockFilePath, trashPath);
+          renamed = true;
           const trashContent = await fs.readFile(trashPath, 'utf-8');
           const trashParsed = this._parseLockContent(trashContent);
           if ((trashParsed?.uuid ?? '') !== expectedUuid) {
             // Lock changed hands mid-check — restore it and back off.
-            try { await fs.link(trashPath, this.lockFilePath); } catch { /* ignore */ }
-            await fs.unlink(trashPath);
+            await this._restoreLockFile(trashPath);
             return;
           }
-          await fs.unlink(trashPath);
+          await fs.unlink(trashPath).catch(() => { /* leaked copy; the sweep above collects it */ });
           logger.log('[FileLockService] Stale lock removed');
-        } catch { /* best-effort startup cleanup; acquire() handles contention */ }
+        } catch {
+          // A failure AFTER the rename means the verify never ran — restore
+          // rather than leave the (possibly live) lock destroyed in the trash.
+          if (renamed) await this._restoreLockFile(trashPath);
+          /* otherwise best-effort startup cleanup; acquire() handles contention */
+        }
       }
     } catch (error: unknown) {
       // ENOENT is expected (no lock file exists)
@@ -232,6 +416,9 @@ export class FileLockService implements IService {
    * @throws Error if unable to acquire lock within timeout
    */
   async acquireLock(): Promise<void> {
+    if (this._retired) {
+      throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service has been disposed`);
+    }
     const heldLocks = lockContext.getStore();
     const lockKey = `${this.lockFilePath}:${this.lockUuid}`;
 
@@ -270,6 +457,14 @@ export class FileLockService implements IService {
       return;
     }
 
+    // Re-check after the wait: cleanup() may have retired the instance while we
+    // were queued. Hand the turn straight on rather than acquiring a lock file this
+    // instance no longer owns.
+    if (this._retired) {
+      myResolve();
+      throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service was disposed while queued`);
+    }
+
     // Capture resolve function so we can trigger it in releaseLock() or on failure
     this.resolveTurn = myResolve;
 
@@ -282,6 +477,15 @@ export class FileLockService implements IService {
 
     try {
       for (let _attempt = 0; _attempt < this.lockRetries; _attempt++) {
+        // cleanup() can retire the instance while we are contending here — its
+        // drain polls only the HELD handle, so a caller mid-retry (holding
+        // nothing yet) is invisible to it and would otherwise win the lock and
+        // run its critical section against torn-down services after dispose
+        // resolved. The queued-waiter check above cannot catch this: our turn was
+        // granted before retirement. Abort; the catch below hands the turn on.
+        if (this._retired) {
+          throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service was disposed during acquisition`);
+        }
         // Use a LOCAL handle variable so that concurrent acquireLock() calls on
         // the same instance cannot close each other's file handle through the
         // shared this.lockHandle field. We only promote to this.lockHandle after
@@ -293,9 +497,20 @@ export class FileLockService implements IService {
           await handle.write(this._makeLockContent());
           await handle.sync(); // Ensure content is written to disk
 
+          // Retirement may have landed between the open above and this commit.
+          // cleanup()'s drain saw no held handle and already returned, so nothing
+          // would ever tear this acquisition down. Throwing here lets the catch
+          // below close the handle and remove the just-created lock file.
+          if (this._retired) {
+            throw new Error(`Cannot acquire lock at ${this.lockFilePath}: service was disposed during acquisition`);
+          }
+
           // Fully acquired — commit to instance state
           this.lockHandle = handle;
           this.lockCount = 1;
+          // Keep the held lock's mtime fresh so a long critical section is not
+          // mistaken for a wedged owner and stolen (see _startHeartbeat).
+          this._startHeartbeat();
 
           // Track in ALS for nested calls
           if (heldLocks) {
@@ -305,7 +520,7 @@ export class FileLockService implements IService {
           const duration = Date.now() - startTime;
           metrics.observe('state_lock_acquire_duration_ms', duration);
           metrics.increment('state_lock_acquire_total', 1, { status: 'success' });
-          metrics.setGauge('state_lock_held', 1);
+          metrics.setGauge('state_lock_held', 1, { lock: this.lockName });
           if (contentionCount > 0) {
             metrics.increment('state_lock_contention_total', 1);
             metrics.observe('state_lock_contention_retries', contentionCount);
@@ -355,26 +570,47 @@ export class FileLockService implements IService {
                 // surrounding retry loop / timeout, not by stealing.
                 if (await this._shouldReclaim(parsed?.pid ?? null, parsed?.startTime ?? null, lockAge, contendedOwnerAliveMemo)) {
                   const trashPath = `${this.lockFilePath}.trash.${crypto.randomBytes(8).toString('hex')}`;
+                  let renamed = false;
                   try {
                     await fs.rename(this.lockFilePath, trashPath);
+                    renamed = true;
                     const trashContent = await fs.readFile(trashPath, 'utf-8');
                     const trashParsed = this._parseLockContent(trashContent);
                     if ((trashParsed?.uuid ?? '') !== lockUuid) {
-                      try { await fs.link(trashPath, this.lockFilePath); } catch { /* ignore */ }
-                      await fs.unlink(trashPath);
+                      await this._restoreLockFile(trashPath);
                       continue;
                     }
-                    await fs.unlink(trashPath);
-                  } catch { /* ignore */ }
+                    await fs.unlink(trashPath).catch(() => { /* leaked copy; startup sweep collects it */ });
+                  } catch {
+                    // Rename failed (nothing moved) — retry handles it. But a read
+                    // failure AFTER the rename means the verify never ran: the lock
+                    // may have changed hands to a live peer mid-check, and leaving
+                    // it in the trash destroys it unverified — the same transient
+                    // classes (EMFILE, EACCES, EBUSY) the inspect branch below
+                    // backs off for. Put it back instead.
+                    if (renamed) await this._restoreLockFile(trashPath);
+                  }
                   continue;
                 }
-              } catch (_statError) {
-                const trashPath = `${this.lockFilePath}.trash.${crypto.randomBytes(8).toString('hex')}`;
-                try {
-                  await fs.rename(this.lockFilePath, trashPath);
-                  await fs.unlink(trashPath);
-                  continue;
-                } catch { /* ignore */ }
+              } catch (inspectError) {
+                // We could not READ the lock, which says nothing about whether its
+                // owner is alive. Destroying it here — as this branch used to,
+                // with neither a UUID check nor a liveness check, unlike the
+                // verified path above — turns any transient fs failure (EMFILE
+                // under fd pressure, EACCES, EBUSY on Windows) into a stolen lock
+                // and two concurrent writers on the same file.
+                //
+                // ENOENT is the exception and the common case: the lock was
+                // released between our open() and our read, so there is nothing to
+                // destroy and retrying immediately is correct.
+                const code = (inspectError as NodeJS.ErrnoException | undefined)?.code;
+                if (code === 'ENOENT') continue;
+                logger.warn(
+                  `[FileLockService] Could not inspect contended lock at ${this.lockFilePath} ` +
+                  `(${code ?? 'unknown'}); backing off rather than reclaiming it.`,
+                );
+                metrics.increment('state_lock_inspect_failed_total', 1, { code: code ?? 'unknown' });
+                // Fall through to the timeout check + backoff below.
               }
 
               if (Date.now() - startTime >= this.lockTimeout) {
@@ -400,10 +636,7 @@ export class FileLockService implements IService {
     } catch (err) {
       // If acquisition failed (timeout/error), we MUST release our turn in the 
       // queue so the next caller can try.
-      if (this.resolveTurn) {
-        this.resolveTurn();
-        this.resolveTurn = null;
-      }
+      this._releaseTurn();
       throw err;
     }
   }
@@ -413,6 +646,11 @@ export class FileLockService implements IService {
    */
   async releaseLock(): Promise<void> {
     if (this.lockHandle === null) {
+      // cleanup() can tear the handle down underneath an in-flight critical
+      // section, so withLock()'s finally lands here with the handle already gone.
+      // The queue turn is still ours to hand back — returning without doing so
+      // leaves every queued waiter blocked on `await previous` for good.
+      this._releaseTurn();
       return;
     }
 
@@ -421,6 +659,10 @@ export class FileLockService implements IService {
       return;
     }
 
+    // Fully releasing (not a re-entrant pop) — the mtime heartbeat must stop
+    // with the hold it maintains.
+    this._stopHeartbeat();
+
     try {
       // 1. Double check ownership before unlinking
       const content = await fs.readFile(this.lockFilePath, 'utf8').catch(() => '');
@@ -440,14 +682,10 @@ export class FileLockService implements IService {
         heldLocks.delete(`${this.lockFilePath}:${this.lockUuid}`);
       }
 
-      metrics.setGauge('state_lock_held', 0);
+      metrics.setGauge('state_lock_held', 0, { lock: this.lockName });
     } finally {
       // 4. Signal the next caller in the queue that it's their turn.
-      const resolve = this.resolveTurn;
-      this.resolveTurn = null;
-      if (resolve) {
-        resolve();
-      }
+      this._releaseTurn();
     }
   }
 
@@ -455,8 +693,13 @@ export class FileLockService implements IService {
    * Execute a function within the scope of a filesystem lock.
    * This is the preferred way to use the lock service.
    * Safe re-entrancy is supported within the same async flow.
+   *
+   * The acquire timeout is the one given to the constructor (`lockTimeout`). This
+   * used to take a second `_timeout` argument that was accepted and then ignored
+   * entirely — a caller passing 5000 silently got the constructed 20000 — so it is
+   * gone rather than left to mislead. No caller was passing it.
    */
-  async withLock<T>(callback: () => Promise<T> | T, _timeout: number = 30000): Promise<T> {
+  async withLock<T>(callback: () => Promise<T> | T): Promise<T> {
     const heldLocks = lockContext.getStore();
     const lockKey = `${this.lockFilePath}:${this.lockUuid}`;
 
@@ -477,9 +720,31 @@ export class FileLockService implements IService {
   }
 
   /**
-   * Force clean up any lock files owned by this instance.
+   * Clean up any lock file owned by this instance, and retire the instance.
+   *
+   * Called only from shutdown paths (`dispose`, `StateManager.cleanup`). It first
+   * gives an in-flight critical section a bounded chance to release normally, then
+   * forces the teardown if it has not. After this the instance is retired:
+   * `acquireLock` fails fast rather than blocking on a queue nothing will ever
+   * drain, which is what makes forcing the teardown safe to do at all.
    */
   async cleanup(): Promise<void> {
+    const drained = await this._drainHeldLock(CLEANUP_DRAIN_TIMEOUT_MS);
+    if (!drained) {
+      logger.warn(
+        `[FileLockService] Lock at ${this.lockFilePath} still held after ${CLEANUP_DRAIN_TIMEOUT_MS}ms; forcing teardown. ` +
+        'A concurrent writer in another process may observe the lock as free.',
+      );
+      metrics.increment('state_lock_forced_teardown_total', 1);
+    }
+
+    // Retire BEFORE releasing the turn, so a waiter woken below re-checks and
+    // fails fast instead of acquiring a lock this instance no longer manages.
+    this._retired = true;
+    // The forced-teardown path bypasses releaseLock, so stop the heartbeat here
+    // too — a retired instance must not keep freshening a lock it forfeited.
+    this._stopHeartbeat();
+
     if (this.lockHandle) {
       try {
         const content = await fs.readFile(this.lockFilePath, 'utf8').catch(() => '');
@@ -494,6 +759,8 @@ export class FileLockService implements IService {
       this.lockHandle = null;
     }
     this.lockCount = 0;
+    // Unblock anyone parked on `await previous`; they will now see _retired and throw.
+    this._releaseTurn();
   }
 
   /**

package/src/infrastructure/knowledge-store-service.ts

@@ -108,10 +108,40 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
       return this._initializationPromise;
     }
 
+    // A different-cwd (or post-dispose-gate) call while an init is still in flight
+    // must NOT start a second closure: the two would race assignments to
+    // _embedder/_store/_writerQueue (the loser's components leak un-disposed), and
+    // the first to settle would null the second's promise marker — letting a
+    // concurrent dispose() skip its wait-for-init guard and be resurrected when the
+    // straggler flips lifecycle back to INITIALIZED. Settle the in-flight init
+    // first, then fall through to the normal re-init path.
+    while (this._initializationPromise) {
+      await this._initializationPromise.catch(() => { /* its own catch reset state */ });
+      // A newer init may have started while we awaited (or while dispose() below
+      // yielded); if it covers our cwd, join it — the loop re-checks either way.
+      if (this._initializationPromise && this._cwd === newCwd) {
+        return this._initializationPromise;
+      }
+      if (this.lifecycle === ServiceLifecycle.INITIALIZED || this.lifecycle === ServiceLifecycle.DISABLED) {
+        if (this._cwd !== newCwd) {
+          logger.log(`[KnowledgeStoreService] CWD changed from ${this._cwd} to ${newCwd}. Re-initializing store...`);
+          await this.dispose();
+        } else {
+          // The settled init already covers this cwd.
+          return;
+        }
+      }
+    }
+
     this.lifecycle = ServiceLifecycle.INITIALIZING;
     logger.debug('[KnowledgeStoreService] Initializing...');
 
-    this._initializationPromise = (async () => {
+    // `let ... = null` (not const): if the closure throws BEFORE its first await,
+    // the finally runs synchronously during the IIFE call — a const self-reference
+    // would be a TDZ crash. The null guard in the finally covers that window (the
+    // marker was never published, so there is nothing to clear).
+    let initPromise: Promise<void> | null = null;
+    initPromise = (async () => {
       try {
         // Create the knowledge store components.
         // Prefer an explicit ctx.cwd; otherwise keep the cwd already resolved on this
@@ -228,11 +258,18 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         this.lifecycle = ServiceLifecycle.UNINITIALIZED;
         throw err;
       } finally {
-        this._initializationPromise = null;
+        // Identity-guarded (same pattern as embedding-factory's clearIfCurrent): only
+        // the promise that owns the marker may null it, so a settling straggler can
+        // never disarm a newer init's marker — which dispose() relies on to wait out
+        // in-flight work.
+        if (this._initializationPromise === initPromise) {
+          this._initializationPromise = null;
+        }
       }
     })();
+    this._initializationPromise = initPromise;
 
-    return this._initializationPromise;
+    return initPromise;
   }
 
   async dispose(): Promise<void> {
@@ -267,6 +304,14 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
 
       if (this._embedder) {
         await this._embedder.dispose?.();
+        // The embedding factory's module-level cache still points at the instance
+        // just disposed — its fast path has no liveness check, so a later re-init
+        // (cwd/mode re-scope) would be handed the dead instance and burn every
+        // warm-up/init retry on it. clearEmbeddingInstance() re-disposes
+        // (idempotent) and nulls the cache; when this process was the leader, its
+        // shutdown deregisters via the serverId CAS, and a client instance's
+        // dispose is a no-op — a FOREIGN leader's registration is never touched.
+        await clearEmbeddingInstance();
       }
 
       if (this._initLock) {