@lincoln504/pi-research 1.2.0 → 1.3.0

package/src/infrastructure/embedding/embedding-server.ts

@@ -16,6 +16,7 @@ import { Utf8Body } from '../../utils/http-body.ts';
 import type { IEmbedder } from '../../core/interfaces/knowledge-interfaces.ts';
 import type { IStateManager } from '../../core/interfaces/state-manager-interfaces.ts';
 import type { Embedder } from '../../knowledge/embedder.ts';
+import { markWebGpuFallback } from '../../knowledge/embedder-utils.ts';
 
 /**
  * Per-process shared secret every embed request must present.
@@ -49,6 +50,24 @@ function isAuthorized(presented: string | string[] | undefined): boolean {
   return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
 }
 
+/**
+ * Refuse to serialize a vector containing NaN/±Infinity. JSON.stringify emits
+ * `null` for non-finite numbers, and the client's `new Float32Array(resp.data)`
+ * coerces null back to 0 — a silently part-zeroed vector that passes every
+ * downstream width/finite gate (0 IS finite) and lands in the ANN index. Failing
+ * the request instead surfaces it as an ordinary embed failure on the client.
+ */
+function assertFiniteVector(vec: ArrayLike<number>, vectorIndex?: number): void {
+  for (let i = 0; i < vec.length; i++) {
+    if (!Number.isFinite(vec[i]!)) {
+      const where = vectorIndex === undefined ? `position ${i}` : `vector ${vectorIndex}, position ${i}`;
+      throw new Error(
+        `[EmbeddingServer] embedder produced a non-finite value (${vec[i]}) at ${where} — refusing to serialize (JSON would launder it to null/0)`,
+      );
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------
 // SerialQueue — ensures embed/embedMany never run concurrently on the GPU
 // ---------------------------------------------------------------------------
@@ -274,6 +293,15 @@ export class EmbeddingServer implements IEmbedder {
       'Stepping down as leader so a fresh process can re-elect with a clean GPU context. ' +
       err.message,
     );
+    // INVARIANT: after a poisoned (permanently hung) inference, THIS process must
+    // never touch WebGPU again. The wedged native call still holds the same-PID
+    // re-entrant GPU lock, so if this process wins the next election and
+    // re-initializes WebGPU it would run a second inference beside the stuck one —
+    // the concurrent-GPU-inference segfault class the SerialQueue exists to
+    // prevent. Latch the process-wide fallback so any embedder built in this
+    // process from here on comes up on CPU; only a FRESH process may re-elect
+    // with a clean GPU context.
+    markWebGpuFallback();
     // Step down WITHOUT disposing the embedder: pipeline.dispose() would join the
     // stuck native thread (itself hanging) or touch a device-lost context. Clearing
     // the state registration + closing the HTTP server is enough for re-election;
@@ -339,14 +367,14 @@ export class EmbeddingServer implements IEmbedder {
       this.leadershipTimer = null;
     }
 
-    // Clear our registration if we still own it
+    // Clear our registration if we still own it. The serverId CAS runs inside the
+    // state lock, so a successor that claimed the slot between our last check and
+    // this shutdown can never be deregistered by us (the old read-then-clear had
+    // exactly that window).
     try {
-      const serverInfo = await this.stateManager.getEmbeddingServer();
-      if (serverInfo?.serverId === this.serverId) {
-        await this.stateManager.clearEmbeddingServer().catch((err) => {
-          logger.warn('[EmbeddingServer] Failed to clear embedding server state:', err);
-        });
-      }
+      await this.stateManager.clearEmbeddingServer({ serverId: this.serverId }).catch((err) => {
+        logger.warn('[EmbeddingServer] Failed to clear embedding server state:', err);
+      });
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
       // State manager is torn down before the embedding server during process exit —
@@ -552,6 +580,9 @@ export class EmbeddingServer implements IEmbedder {
               // message drives the client to reconnect to the new leader.
               this.assertServing();
               const result = await this.queue.enqueue(() => this.embedder.embed(data.text as string));
+              // Scan BEFORE serialization: a device-lost mid-inference yields NaN
+              // entries that JSON would launder into null → 0 on the client.
+              assertFiniteVector(result);
               res.writeHead(200, { 'Content-Type': 'application/json' });
               res.end(JSON.stringify({ data: Array.from(result) }));
               break;
@@ -559,6 +590,8 @@ export class EmbeddingServer implements IEmbedder {
             case '/embedMany': {
               this.assertServing();
               const results = await this.queue.enqueue(() => this.embedder.embedMany(data.texts as string[]));
+              // Same non-finite scan as /embed, per vector (see assertFiniteVector).
+              for (let i = 0; i < results.length; i++) assertFiniteVector(results[i]!, i);
               res.writeHead(200, { 'Content-Type': 'application/json' });
               res.end(JSON.stringify({ results: results.map(r => Array.from(r)) }));
               break;

package/src/infrastructure/file-lock-service.ts

@@ -38,6 +38,16 @@ const CLEANUP_DRAIN_TIMEOUT_MS = 5_000;
  */
 const TRASH_SWEEP_AGE_MS = 10 * 60_000;
 
+/**
+ * Floor for the held-lock mtime heartbeat interval (see _startHeartbeat), so a
+ * small liveOwnerStaleThreshold cannot turn maintenance into a utimes storm.
+ */
+const HEARTBEAT_MIN_INTERVAL_MS = 2_000;
+
+/** setTimeout/setInterval clamp delays above 2^31-1 to 1ms — treat anything
+ *  bigger as "no heartbeat" rather than a busy-loop. */
+const MAX_TIMER_DELAY_MS = 0x7fffffff;
+
 /**
  * Lock configuration options
  */
@@ -101,9 +111,17 @@ export class FileLockService implements IService {
   // Set by cleanup(). A retired instance no longer manages the lock file, so
   // acquiring through it would hand out exclusivity it cannot honour.
   private _retired = false;
+  // Refreshes the held lock file's mtime while a critical section runs (see
+  // _startHeartbeat). Null whenever no lock is held.
+  private heartbeatTimer: NodeJS.Timeout | null = null;
+  // Stable short lock identity for metrics labels — without it every instance
+  // (state lock, knowledge init, browser init, run slots) shares one gauge and
+  // any release zeroes it globally.
+  private readonly lockName: string;
 
   constructor(options: FileLockOptions) {
     this.lockFilePath = options.lockFilePath;
+    this.lockName = pathmod.basename(options.lockFilePath);
     this.lockTimeout = options.lockTimeout ?? 20000;
     this.lockRetries = options.lockRetries ?? 200;
     this.lockRetryDelay = options.lockRetryDelay ?? 100;
@@ -254,6 +272,48 @@ export class FileLockService implements IService {
     if (resolve) resolve();
   }
 
+  /**
+   * While the lock is HELD, refresh its mtime on an interval well inside
+   * liveOwnerStaleThreshold. Peers judge a live owner purely by lock-file age
+   * (_shouldReclaim), and nothing else touches the file during a critical
+   * section — so without a heartbeat a legitimately long one (e.g. a multi-minute
+   * knowledge-store model-change migration under the 240s init-lock threshold)
+   * ages past the limit and is stolen from a LIVE owner, admitting a concurrent
+   * writer. With the heartbeat, a live owner is only stolen after genuinely
+   * wedging: no refresh for the full threshold.
+   *
+   * unref() is CORRECT here, unlike a timer that IS the operation: the heartbeat
+   * is maintenance on behalf of a critical section that keeps the process alive
+   * by itself, and the process must never be kept alive just to freshen a lock.
+   *
+   * The refresh goes through the held FileHandle (futimes), not the path, so if
+   * the lock changed hands (renamed/unlinked by a reclaiming peer) we can only
+   * ever touch OUR old inode — never freshen a peer's lock by mistake.
+   */
+  private _startHeartbeat(): void {
+    this._stopHeartbeat();
+    const interval = Math.max(HEARTBEAT_MIN_INTERVAL_MS, Math.ceil(this.liveOwnerStaleThreshold / 4));
+    // An effectively-infinite threshold (the run-semaphore's never-steal-from-a-
+    // live-owner opt-out) needs no heartbeat — and would overflow the 32-bit
+    // timer delay, which Node clamps to 1ms (a utimes busy-loop). Skip entirely;
+    // the opt-out's semantics are unchanged.
+    if (interval > MAX_TIMER_DELAY_MS) return;
+    this.heartbeatTimer = setInterval(() => {
+      const handle = this.lockHandle;
+      if (handle === null) return;
+      const now = new Date();
+      handle.utimes(now, now).catch(() => { /* best-effort maintenance */ });
+    }, interval);
+    this.heartbeatTimer.unref?.();
+  }
+
+  private _stopHeartbeat(): void {
+    if (this.heartbeatTimer !== null) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
+
   /**
    * Wait, bounded, for an in-flight critical section to finish.
    *
@@ -448,6 +508,9 @@ export class FileLockService implements IService {
           // Fully acquired — commit to instance state
           this.lockHandle = handle;
           this.lockCount = 1;
+          // Keep the held lock's mtime fresh so a long critical section is not
+          // mistaken for a wedged owner and stolen (see _startHeartbeat).
+          this._startHeartbeat();
 
           // Track in ALS for nested calls
           if (heldLocks) {
@@ -457,7 +520,7 @@ export class FileLockService implements IService {
           const duration = Date.now() - startTime;
           metrics.observe('state_lock_acquire_duration_ms', duration);
           metrics.increment('state_lock_acquire_total', 1, { status: 'success' });
-          metrics.setGauge('state_lock_held', 1);
+          metrics.setGauge('state_lock_held', 1, { lock: this.lockName });
           if (contentionCount > 0) {
             metrics.increment('state_lock_contention_total', 1);
             metrics.observe('state_lock_contention_retries', contentionCount);
@@ -596,6 +659,10 @@ export class FileLockService implements IService {
       return;
     }
 
+    // Fully releasing (not a re-entrant pop) — the mtime heartbeat must stop
+    // with the hold it maintains.
+    this._stopHeartbeat();
+
     try {
       // 1. Double check ownership before unlinking
       const content = await fs.readFile(this.lockFilePath, 'utf8').catch(() => '');
@@ -615,7 +682,7 @@ export class FileLockService implements IService {
         heldLocks.delete(`${this.lockFilePath}:${this.lockUuid}`);
       }
 
-      metrics.setGauge('state_lock_held', 0);
+      metrics.setGauge('state_lock_held', 0, { lock: this.lockName });
     } finally {
       // 4. Signal the next caller in the queue that it's their turn.
       this._releaseTurn();
@@ -674,6 +741,9 @@ export class FileLockService implements IService {
     // Retire BEFORE releasing the turn, so a waiter woken below re-checks and
     // fails fast instead of acquiring a lock this instance no longer manages.
     this._retired = true;
+    // The forced-teardown path bypasses releaseLock, so stop the heartbeat here
+    // too — a retired instance must not keep freshening a lock it forfeited.
+    this._stopHeartbeat();
 
     if (this.lockHandle) {
       try {

package/src/infrastructure/knowledge-store-service.ts

@@ -108,10 +108,40 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
       return this._initializationPromise;
     }
 
+    // A different-cwd (or post-dispose-gate) call while an init is still in flight
+    // must NOT start a second closure: the two would race assignments to
+    // _embedder/_store/_writerQueue (the loser's components leak un-disposed), and
+    // the first to settle would null the second's promise marker — letting a
+    // concurrent dispose() skip its wait-for-init guard and be resurrected when the
+    // straggler flips lifecycle back to INITIALIZED. Settle the in-flight init
+    // first, then fall through to the normal re-init path.
+    while (this._initializationPromise) {
+      await this._initializationPromise.catch(() => { /* its own catch reset state */ });
+      // A newer init may have started while we awaited (or while dispose() below
+      // yielded); if it covers our cwd, join it — the loop re-checks either way.
+      if (this._initializationPromise && this._cwd === newCwd) {
+        return this._initializationPromise;
+      }
+      if (this.lifecycle === ServiceLifecycle.INITIALIZED || this.lifecycle === ServiceLifecycle.DISABLED) {
+        if (this._cwd !== newCwd) {
+          logger.log(`[KnowledgeStoreService] CWD changed from ${this._cwd} to ${newCwd}. Re-initializing store...`);
+          await this.dispose();
+        } else {
+          // The settled init already covers this cwd.
+          return;
+        }
+      }
+    }
+
     this.lifecycle = ServiceLifecycle.INITIALIZING;
     logger.debug('[KnowledgeStoreService] Initializing...');
 
-    this._initializationPromise = (async () => {
+    // `let ... = null` (not const): if the closure throws BEFORE its first await,
+    // the finally runs synchronously during the IIFE call — a const self-reference
+    // would be a TDZ crash. The null guard in the finally covers that window (the
+    // marker was never published, so there is nothing to clear).
+    let initPromise: Promise<void> | null = null;
+    initPromise = (async () => {
       try {
         // Create the knowledge store components.
         // Prefer an explicit ctx.cwd; otherwise keep the cwd already resolved on this
@@ -228,11 +258,18 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         this.lifecycle = ServiceLifecycle.UNINITIALIZED;
         throw err;
       } finally {
-        this._initializationPromise = null;
+        // Identity-guarded (same pattern as embedding-factory's clearIfCurrent): only
+        // the promise that owns the marker may null it, so a settling straggler can
+        // never disarm a newer init's marker — which dispose() relies on to wait out
+        // in-flight work.
+        if (this._initializationPromise === initPromise) {
+          this._initializationPromise = null;
+        }
       }
     })();
+    this._initializationPromise = initPromise;
 
-    return this._initializationPromise;
+    return initPromise;
   }
 
   async dispose(): Promise<void> {
@@ -267,6 +304,14 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
 
       if (this._embedder) {
         await this._embedder.dispose?.();
+        // The embedding factory's module-level cache still points at the instance
+        // just disposed — its fast path has no liveness check, so a later re-init
+        // (cwd/mode re-scope) would be handed the dead instance and burn every
+        // warm-up/init retry on it. clearEmbeddingInstance() re-disposes
+        // (idempotent) and nulls the cache; when this process was the leader, its
+        // shutdown deregisters via the serverId CAS, and a client instance's
+        // dispose is a no-op — a FOREIGN leader's registration is never touched.
+        await clearEmbeddingInstance();
       }
 
       if (this._initLock) {

package/src/infrastructure/process-lifecycle-service.ts

@@ -55,6 +55,17 @@ export class ProcessLifecycleService implements IProcessLifecycle {
 
   private cachedStartTime: number | null = null;
 
+  // Short-TTL memo for SUBPROCESS-backed start-time lookups (macOS `ps`, Windows
+  // powershell — Linux reads /proc and stays uncached). A queued run-slot waiter
+  // sweeps every 250ms and builds a fresh FileLockService per attempt, so without
+  // a cross-attempt memo each live owner costs up to two `ps` spawns per sweep —
+  // ~48 subprocess spawns/second sustained on macOS while queued. A live PID's
+  // start time never changes; the only staleness is a PID recycled within the
+  // TTL reading as its dead predecessor, which merely delays a stale-owner steal
+  // by up to the TTL (well under every staleness threshold in use).
+  private static readonly START_TIME_TTL_MS = 5000;
+  private startTimeMemo = new Map<number, { value: number | null; at: number }>();
+
   async initialize(): Promise<void> {
     this.lifecycle = ServiceLifecycle.INITIALIZED;
   }
@@ -87,6 +98,18 @@ export class ProcessLifecycleService implements IProcessLifecycle {
       }
     }
 
+    const memoized = this.startTimeMemo.get(pid);
+    if (memoized && Date.now() - memoized.at < ProcessLifecycleService.START_TIME_TTL_MS) {
+      return memoized.value;
+    }
+    const memoize = (value: number | null): number | null => {
+      // Bound the map: sweeps interrogate a handful of owners; anything beyond a
+      // few hundred distinct PIDs inside one TTL window means leaked entries.
+      if (this.startTimeMemo.size > 256) this.startTimeMemo.clear();
+      this.startTimeMemo.set(pid, { value, at: Date.now() });
+      return value;
+    };
+
     // Windows: use powershell to get process creation time
     if (process.platform === 'win32') {
       try {
@@ -99,11 +122,11 @@ export class ProcessLifecycleService implements IProcessLifecycle {
           );
           child.unref?.();
         });
-        if (!output || !output.trim()) return null;
+        if (!output || !output.trim()) return memoize(null);
         const startTime = parseInt(output.trim(), 10);
-        return isNaN(startTime) ? null : startTime;
+        return memoize(isNaN(startTime) ? null : startTime);
       } catch {
-        return null;
+        return memoize(null);
       }
     }
 
@@ -121,8 +144,8 @@ export class ProcessLifecycleService implements IProcessLifecycle {
     // So: try `etimes`, and fall back to parsing POSIX `etime` when it yields
     // nothing. `etime` is mandated by POSIX, so the fallback works everywhere.
     const elapsedSec = await this.readElapsedSeconds(pid);
-    if (elapsedSec === null) return null;
-    return Math.floor(Date.now() / 1000) - elapsedSec;
+    if (elapsedSec === null) return memoize(null);
+    return memoize(Math.floor(Date.now() / 1000) - elapsedSec);
   }
 
   /** Elapsed seconds for `pid` via ps, or null. Tries `etimes`, then POSIX `etime`. */

package/src/infrastructure/research-run-semaphore.ts

@@ -195,7 +195,18 @@ export class ResearchRunSemaphore implements IService {
     try {
       await slot.initialize(); // resolves own startTime (cached) + startup stale cleanup
       await slot.acquireLock(); // try-take: wins, reclaims a dead owner, or throws fast
-    } catch {
+    } catch (err) {
+      // Contention and breakage are different outcomes and must not be conflated.
+      // FileLockService synthesizes its own Errors (no errno code) for the two
+      // contention shapes a try-take produces — a live-contended slot ("timeout
+      // after 0ms", cause EEXIST) and a lost reclaim race ("after N retries") —
+      // and those genuinely mean "slot busy". A real I/O failure (ENOENT/EACCES/
+      // ENOSPC: the slot file is uncreatable; both mkdirs are warn-only) instead
+      // propagates the raw errno error. Swallowing that as "busy" would poll out
+      // the whole acquire window and reject with a FALSE ResearchRunCapacityError,
+      // making the documented fail-open contract (research proceeds when the
+      // semaphore is broken) unreachable. Rethrow so acquire() can tell them apart.
+      if (typeof (err as NodeJS.ErrnoException | null)?.code === 'string') throw err;
       return null; // busy (live owner) or lost a reclaim race — try the next slot
     }
     return {
@@ -246,16 +257,39 @@ export class ResearchRunSemaphore implements IService {
       // Round-robin from a rotating start so contending processes don't synchronize on slot 0.
       const start = this.nextStartIndex;
       this.nextStartIndex = (this.nextStartIndex + 1) % this.maxSlots;
+      // Slot BREAKAGE (I/O error) is tracked separately from contention within
+      // this sweep — a broken slot must never be reported as a busy one.
+      let brokenSlots = 0;
+      let firstBreakage: unknown = null;
       for (let offset = 0; offset < this.maxSlots; offset++) {
         const i = (start + offset) % this.maxSlots;
-        const got = await this.trySlot(i);
+        let got: RunSlotAcquisition | null;
+        try {
+          got = await this.trySlot(i);
+        } catch (err) {
+          // This slot is broken, not busy. Keep sweeping — another slot may still
+          // be acquirable (mixed case: slot 0 uncreatable, slot 1 free).
+          brokenSlots++;
+          firstBreakage ??= err;
+          logger.debug(`[ResearchRunSemaphore] Slot ${i} errored (broken, not busy): ${err instanceof Error ? err.message : String(err)}`);
+          continue;
+        }
         if (got) {
           metrics.setGauge('research_run_slots_held', 1, { slot: String(i) });
           metrics.increment('research_run_acquire_total', 1, { status: 'acquired' });
           return got;
         }
       }
-      // Every slot is held by a live process.
+      // EVERY slot errored: the semaphore itself is broken, not at capacity.
+      // Waiting cannot help — nothing is held that could free up — and a
+      // ResearchRunCapacityError here would misclassify breakage as retryable
+      // contention while the caller's fail-open path (research proceeds without
+      // the cap) stays unreachable. Rethrow the I/O error instead.
+      if (brokenSlots === this.maxSlots) {
+        metrics.increment('research_run_acquire_total', 1, { status: 'error' });
+        throw firstBreakage;
+      }
+      // Every remaining slot is held by a live process.
       if (Date.now() >= deadline) {
         metrics.increment('research_run_acquire_total', 1, { status: 'exhausted' });
         throw new ResearchRunCapacityError(this.maxSlots, maxWaitMs);

package/src/infrastructure/state/state-browser-api.ts

@@ -50,12 +50,15 @@ export class StateBrowserApi {
   /**
    * Clear the browser server information
    * @param updateState Function to update state atomically
+   * @param expected Optional expected-owner identity forwarded to the CAS check
+   *   in StateBrowserManager.clearBrowserServer.
    */
   async clearBrowserServer(
-    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>
+    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>,
+    expected?: { pid?: number; schedulerId?: string }
   ): Promise<void> {
     await updateState((state) => {
-      return this.browserManager.clearBrowserServer(state);
+      return this.browserManager.clearBrowserServer(state, expected);
     });
   }
 

package/src/infrastructure/state/state-browser-manager.ts

@@ -47,9 +47,23 @@ export class StateBrowserManager implements IService {
   /**
    * Clear the browser server information
    * @param state The current state
+   * @param expected Optional identity the caller believes is currently registered.
+   *   When supplied, this is a compare-and-delete: the entry is only removed if it
+   *   still matches (mirrors StateManager.clearEmbeddingServer). browserServer is
+   *   shared cross-process state, and an unconditional delete lets a caller who
+   *   decided the *previous* leader was dead deregister a DIFFERENT process's
+   *   fresh, live registration that claimed the slot in the meantime — that new
+   *   leader then runs undiscoverable (state shows no browserServer), so the next
+   *   getScheduler() caller elects a second, redundant leader. A mismatch means
+   *   someone else already claimed the slot, and the clear must no-op.
    * @returns Updated state without browser server info
    */
-  clearBrowserServer(state: SingletonState): SingletonState {
+  clearBrowserServer(state: SingletonState, expected?: { pid?: number; schedulerId?: string }): SingletonState {
+    const entry = state.browserServer;
+    if (entry && expected !== undefined) {
+      if (expected.schedulerId !== undefined && entry.schedulerId !== expected.schedulerId) return state;
+      if (expected.pid !== undefined && entry.pid !== expected.pid) return state;
+    }
     delete state.browserServer;
     return state;
   }

package/src/infrastructure/state/state-manager-service.ts

@@ -188,8 +188,8 @@ export class StateManagerService implements IStateManager {
   /**
    * Clear the browser server information
    */
-  async clearBrowserServer(): Promise<void> {
-    return this.getStateManager().clearBrowserServer();
+  async clearBrowserServer(expected?: { pid?: number; schedulerId?: string }): Promise<void> {
+    return this.getStateManager().clearBrowserServer(expected);
   }
 
   /**
@@ -202,8 +202,8 @@ export class StateManagerService implements IStateManager {
   /**
    * Clear the embedding server information
    */
-  async clearEmbeddingServer(): Promise<void> {
-    return this.getStateManager().clearEmbeddingServer();
+  async clearEmbeddingServer(expected?: { pid?: number; serverId?: string }): Promise<void> {
+    return this.getStateManager().clearEmbeddingServer(expected);
   }
 
   /**

package/src/infrastructure/state/state-manager.ts

@@ -446,8 +446,8 @@ export class StateManager {
     );
   }
 
-  public async clearBrowserServer(): Promise<void> {
-    await this.browserApi.clearBrowserServer(this.updateState.bind(this));
+  public async clearBrowserServer(expected?: { pid?: number; schedulerId?: string }): Promise<void> {
+    await this.browserApi.clearBrowserServer(this.updateState.bind(this), expected);
   }
 
   // ==================== Embedding Server API ====================
@@ -457,8 +457,20 @@ export class StateManager {
     return state.embeddingServer ?? null;
   }
 
-  public async clearEmbeddingServer(): Promise<void> {
+  public async clearEmbeddingServer(expected?: { pid?: number; serverId?: string }): Promise<void> {
     await this.updateState((state) => {
+      // Compare-and-delete: the registration is shared cross-process state, and an
+      // unconditional delete lets a follower's shutdown (or a waiter acting on a
+      // stale snapshot) deregister a DIFFERENT process's live leader — which then
+      // never re-registers (its leadership check treats an absent entry as benign),
+      // so a fresh caller elects a second GPU model instance alongside the invisible
+      // one. Callers pass the identity they believe they are clearing; a mismatch
+      // means someone else already claimed the slot, and the clear must no-op.
+      const entry = state.embeddingServer;
+      if (entry && expected !== undefined) {
+        if (expected.serverId !== undefined && entry.serverId !== expected.serverId) return state;
+        if (expected.pid !== undefined && entry.pid !== expected.pid) return state;
+      }
       delete state.embeddingServer;
       return state;
     });

package/src/knowledge/chunker.ts

@@ -42,6 +42,7 @@ export class Chunker {
     let spaceExtensions = 0;
 
     while (start < text.length) {
+      const iterationStart = start;
       let end = start + this.targetSize;
       
       if (end < text.length) {
@@ -118,6 +119,55 @@ export class Chunker {
 
       if (end <= start) end = Math.min(start + this.targetSize, text.length);
 
+      // Forward-progress invariant: every chunk after the first must end past the
+      // previous chunk's end. When the overlap step plants `start` inside a code
+      // block whose closing fence terminated the PREVIOUS chunk, the in-block
+      // re-extension above clamps `end` back onto that same fence — a chunk that
+      // re-covers only overlap content, and (via the 1-char anti-regression step
+      // below) one such sliver per character until `start` clears the fence.
+      // Covering fresh text is worth splitting the fence: the previous chunk
+      // already carries the block's tail intact.
+      if (chunks.length > 0 && end <= prevEnd) {
+        end = Math.min(start + this.targetSize, text.length);
+      }
+
+      // UTF-16 surrogate-pair safety: `end` is a raw code-unit index, and every
+      // adjustment above (code-block/heading/sentence/newline/space, plus the
+      // forward-progress guard) operates purely in code-unit space — none of
+      // them are aware of astral characters (emoji, rare CJK Extension B+),
+      // which are a HIGH surrogate (0xD800-0xDBFF) followed by a LOW surrogate
+      // (0xDC00-0xDFFF). If `end` lands between the two halves, slicing here
+      // emits a lone high surrogate. Node's UTF-16->UTF-8 conversion (JSON
+      // serialization, HTTP bodies, N-API marshalling into the LanceDB/Arrow
+      // layer) silently substitutes U+FFFD for that rather than throwing —
+      // silent corruption of text that gets embedded and stored. Mirrors the
+      // backoff in truncateWithMarker (src/utils/text-utils.ts). Applied last,
+      // right before the slice, and only shrinks `end` when doing so can't
+      // undo the forward-progress guard just above or produce an empty chunk.
+      if (end < text.length) {
+        const code = text.charCodeAt(end - 1);
+        if (code >= 0xd800 && code <= 0xdbff) {
+          const candidate = end - 1;
+          const staysPastPrev = chunks.length === 0 || candidate > prevEnd;
+          if (candidate > start && staysPastPrev) {
+            end = candidate;
+          } else {
+            // Backing off would violate the forward-progress guard above (or empty
+            // the chunk) — this happens when targetSize and overlap sit within one
+            // code unit of each other. Grow past the low surrogate instead: taking
+            // the full pair into THIS chunk only ever increases `end`, so it can
+            // never re-collide with the guard it just failed, and it still avoids
+            // splitting the pair (the alternative — leaving `end` right after the
+            // lone high surrogate — is the silent corruption this block exists to
+            // prevent).
+            const low = text.charCodeAt(end);
+            if (low >= 0xdc00 && low <= 0xdfff) {
+              end = Math.min(end + 1, text.length);
+            }
+          }
+        }
+      }
+
       const chunkText = text.slice(start, end);
       const actual_overlap = chunks.length === 0 ? 0 : Math.max(0, Math.min(chunkText.length, prevEnd - start));
       
@@ -131,7 +181,44 @@ export class Chunker {
         start = end - Math.min(this.overlap, chunkText.length - 1);
       }
       if (start < 0) start = 0;
-      if (start >= end) start = end - 1; 
+      if (start >= end) start = end - 1;
+
+      // Forward-progress invariant, start side: the constructor rejects
+      // overlap >= targetSize specifically "to prevent infinite loops," but that
+      // guarantee only holds for the raw arithmetic above — it says nothing about
+      // the end-side forward-progress guard earlier in this loop reusing the same
+      // `end` two iterations in a row, which can leave `start` no greater than the
+      // value this same iteration started with, reproducing an identical chunk
+      // forever (observed: OOM from an unbounded chunks array on a small
+      // targetSize/overlap gap plus a boundary emoji). Guarantee real progress
+      // independent of every heuristic above, mirroring the `end`-side guard.
+      // Applied BEFORE the surrogate-pair check below so that check always
+      // adjusts an already-safe value instead of being able to undo this guard.
+      if (start <= iterationStart) start = iterationStart + 1;
+
+      // Surrogate-pair safety, read side: `start` (end - overlap, further
+      // clamped above) is an independent raw code-unit index — it is not
+      // derived from any of the textual heuristics that picked `end` above,
+      // so the fix to `end` does not cover it. It can separately bisect an
+      // unrelated pair and produce a next chunk that OPENS with a lone low
+      // surrogate. The pair's high half is already inside the chunk just
+      // pushed (it sits before `end`), so pulling `start` back one unit only
+      // widens that chunk's overlap by one code unit — it never drops text.
+      // Only pull back when doing so keeps the forward-progress guard above
+      // satisfied; otherwise push forward past the low surrogate instead — the
+      // pair's high half is guaranteed already present in the chunk just
+      // pushed, so skipping the low half here loses nothing, it just narrows
+      // this chunk's overlap by one code unit instead of widening it.
+      if (start > 0) {
+        const code = text.charCodeAt(start);
+        if (code >= 0xdc00 && code <= 0xdfff) {
+          if (start - 1 > iterationStart) {
+            start -= 1;
+          } else if (start + 1 < end) {
+            start += 1;
+          }
+        }
+      }
     }
 
     const duration = Date.now() - startTime;