@lincoln504/pi-research 1.3.12 → 1.4.1

package/src/infrastructure/embedding/embedding-factory.ts

@@ -227,6 +227,33 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
       const POLL_TIMEOUT_MS = 120_000;
       const deadline = Date.now() + POLL_TIMEOUT_MS;
 
+      /**
+       * How many CONSECUTIVE polls must find a confirmed-live leader's endpoint
+       * unusable before its registration is judged stale.
+       *
+       * Deleting a live leader's entry is the expensive mistake in this loop: the
+       * leader is not told, treats a missing entry as benign, and never steps down,
+       * so the follower's subsequent election produces a second model init and a
+       * second GPU context while the first server keeps running, invisible. The old
+       * code reached that from two failed TCP probes ~2.5s apart — well inside a
+       * blip from a saturated accept backlog, a paused event loop during a large
+       * batch embed, or CPU starvation on a shared box.
+       *
+       * Not clearing at all is not the answer either: a leader whose HTTP server
+       * died while the process lives on leaves a registration nothing else collects,
+       * and every later process then falls back to its own in-process embedder. So
+       * the threshold is raised rather than removed. Each iteration costs the poll
+       * interval plus up to the 2s socket probe (isPortListening) plus up to a
+       * further 2s health-check probe (fetchHealth, called below with an explicit
+       * short timeout — NOT the client's 120s default, which would let a merely-busy
+       * leader blow the outer POLL_TIMEOUT_MS long before this counter ever fired),
+       * making this roughly 45s of uninterrupted unreachability — far past any blip,
+       * far short of the 120s wait.
+       */
+      const UNREACHABLE_POLLS_BEFORE_STALE = 10;
+      let unreachablePolls = 0;
+      let unreachableEndpoint: string | null = null;
+
       logger.info(`[EmbeddingFactory] Another process is initializing (port=${fastPathPort}), waiting...`);
 
       while (Date.now() < deadline) {
@@ -253,24 +280,50 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
           return localFallbackForMismatch(info.model);
         }
         if (info.port > 0) {
-          const portOk = await isPortListening(info.port);
-          if (portOk) {
+          // A new claimant resets the streak — the count must describe ONE endpoint's
+          // continuous unreachability, not a tally across successive leaders.
+          const endpoint = `${info.serverId}:${info.port}`;
+          if (endpoint !== unreachableEndpoint) {
+            unreachableEndpoint = endpoint;
+            unreachablePolls = 0;
+          }
+
+          // Reaching the counter below means "not usable this poll" by construction:
+          // the only usable outcome returns.
+          if (await isPortListening(info.port)) {
             logger.info(`[EmbeddingFactory] Connecting to embedding server on port ${info.port}`);
             const client = new EmbeddingClient(info.port, cfg.EMBEDDING_DEVICE, info.authSecret);
-            await client.fetchHealth();
-            _embeddingInstance = client;
-            _cachedModel = cfg.EMBEDDING_MODEL;
-            _cachedDevice = cfg.EMBEDDING_DEVICE;
-            return client;
+            try {
+              // A short, poll-scale timeout — NOT EmbeddingClient's 120s default. The
+              // unreachable-poll count above assumes each iteration costs roughly the poll
+              // interval plus this probe's timeout (~2.5s total); at the 120s default, a
+              // leader that is merely busy (not dead) can take the full 120s to answer,
+              // blowing the outer POLL_TIMEOUT_MS well before the stale-registration
+              // threshold below would ever fire. Matches isPortListening's own 2s probe.
+              await client.fetchHealth(2_000);
+              _embeddingInstance = client;
+              _cachedModel = cfg.EMBEDDING_MODEL;
+              _cachedDevice = cfg.EMBEDDING_DEVICE;
+              return client;
+            } catch (err) {
+              // The socket accepted but the server did not answer. Previously this
+              // rejected out of getEmbedder entirely, so one failed health request
+              // against an otherwise-recoverable leader (mid model swap, momentarily
+              // wedged) took embedding down for the whole session instead of costing
+              // one poll. Count it as an unreachable poll and keep waiting.
+              logger.debug(`[EmbeddingFactory] Health check against port ${info.port} failed:`, err);
+            }
           }
-          // Port registered but not responding — wait one more interval and re-check
-          await new Promise<void>((r) => setTimeout(r, POLL_INTERVAL_MS));
-          const secondCheck = await isPortListening(info.port);
-          if (!secondCheck) {
-            logger.warn(`[EmbeddingFactory] Registered port ${info.port} is unreachable after two checks — clearing stale state`);
-            // serverId-scoped: this snapshot is up to ~2.5s old (two probes + sleep);
-            // the entry may already belong to a new claimant. See the dead-leader
-            // clear above for the failure this prevents.
+
+          unreachablePolls++;
+          if (unreachablePolls >= UNREACHABLE_POLLS_BEFORE_STALE) {
+            logger.warn(
+              `[EmbeddingFactory] Registered port ${info.port} unreachable for ${unreachablePolls} consecutive polls ` +
+              `(PID ${info.pid} is alive) — treating the registration as stale and electing a replacement.`,
+            );
+            // serverId-scoped: this snapshot is many seconds old by now and the entry
+            // may already belong to a new claimant. See the dead-leader clear above
+            // for the failure an unconditional delete causes.
             await stateManager.clearEmbeddingServer({ serverId: info.serverId }).catch((err) => logger.debug('Swallowed clear embedding server error:', err));
             break;
           }
@@ -355,6 +408,21 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
 // clearEmbeddingInstance — for testing / restart scenarios
 // ---------------------------------------------------------------------------
 
+/**
+ * Does THIS process currently hold a live embedding server (as opposed to a client
+ * pointing at someone else's, or nothing at all)?
+ *
+ * Asked by the teardown-time state clear, which is pid-scoped and therefore cannot
+ * tell "my leftover registration" from "my server that is still listening". That
+ * clear runs before disposeAllServices, so at that moment our own server usually
+ * IS still up — and deleting its entry makes it an invisible leader, since its
+ * leadership check reads a missing entry as benign and never steps down. Its own
+ * dispose clears the entry properly, keyed on serverId.
+ */
+export function ownsLiveEmbeddingServer(): boolean {
+    return _embeddingInstance !== null && _embeddingInstance instanceof EmbeddingServer;
+}
+
 export async function clearEmbeddingInstance(): Promise<void> {
   if (_embeddingInstance) {
     try {

package/src/infrastructure/file-lock-service.ts

@@ -44,6 +44,49 @@ const TRASH_SWEEP_AGE_MS = 10 * 60_000;
  */
 const HEARTBEAT_MIN_INTERVAL_MS = 2_000;
 
+/**
+ * Ceiling on the held-lock mtime heartbeat interval.
+ *
+ * The interval derives from liveOwnerStaleThreshold/4, which is sized so a live
+ * holder is never judged stale between two refreshes. That is the right shape for
+ * the live-owner rule and the wrong size for everything downstream of it: the
+ * knowledge-store init lock derived a 60s interval, and _unidentifiedStaleThreshold
+ * doubles it, so a lock torn by SIGKILL — unidentifiable by construction — was left
+ * alone for two minutes while the acquirer's stall bound gave up after twenty
+ * seconds. The recovery the unidentified rule exists to allow could not happen.
+ *
+ * A utimes every few seconds costs nothing, so cap it. The cap does not change who
+ * may be reclaimed: liveOwnerStaleThreshold is untouched, and refreshing more often
+ * than required only makes a live holder harder to mistake for a dead one.
+ */
+const HEARTBEAT_MAX_INTERVAL_MS = 5_000;
+
+/**
+ * Ceiling on how long a lock whose owner cannot be identified is left alone.
+ *
+ * Such a lock is treated as live (we could not prove otherwise), but "treated as
+ * live" must not mean "immortal" — see _shouldReclaim for why the never-steal
+ * opt-out would otherwise strand a run slot permanently.
+ */
+const UNIDENTIFIED_MAX_STALE_MS = 10 * 60_000;
+
+/**
+ * Absolute ceiling on one acquireLock() wait, however healthy the holder looks.
+ *
+ * `lockTimeout` bounds a STALL — time in which the lock did not change hands and
+ * its holder did not refresh it. It deliberately does not bound a wait behind a
+ * holder that is visibly working, because that turned ordinary contention into
+ * failed runs: the knowledge-store init lock allowed 20s while the same lock
+ * covers a post-run FTS rebuild plus optimize, so a second run starting during
+ * another's housekeeping failed outright. Twice, on 2026-08-14. Widening the
+ * constant is the fix that does not work — housekeeping has no fixed duration.
+ *
+ * This ceiling is what keeps "wait while progress is happening" from meaning
+ * "wait forever" under sustained contention, where the lock keeps changing hands
+ * and every observation looks like progress while this caller never wins.
+ */
+export const LOCK_ACQUIRE_CEILING_MS = 5 * 60_000;
+
 /** setTimeout/setInterval clamp delays above 2^31-1 to 1ms — treat anything
  *  bigger as "no heartbeat" rather than a busy-loop. */
 const MAX_TIMER_DELAY_MS = 0x7fffffff;
@@ -62,6 +105,13 @@ export interface FileLockOptions {
   lockRetryDelay?: number;
   /** Stale lock threshold in milliseconds (default: 15 seconds) */
   lockStaleThreshold?: number;
+  /**
+   * Absolute ceiling on one acquireLock() wait, however healthy the holder looks
+   * (default: LOCK_ACQUIRE_CEILING_MS). `lockTimeout` bounds a stall; this bounds
+   * the total. Exposed so the bound is reachable in a test — a five-minute
+   * backstop that no test can drive is a five-minute backstop nobody has checked.
+   */
+  acquireCeilingMs?: number;
   /**
    * Stale threshold (ms) applied ONLY when the lock's owner process is provably
    * alive. Much larger than lockStaleThreshold so a live process holding the lock
@@ -95,6 +145,7 @@ export class FileLockService implements IService {
   private readonly lockRetryDelay: number;
   private readonly lockStaleThreshold: number;
   private readonly liveOwnerStaleThreshold: number;
+  private readonly acquireCeilingMs: number;
   private readonly processLifecycle: IProcessLifecycle | null;
 
   // Lock tracking
@@ -128,18 +179,41 @@ export class FileLockService implements IService {
     this.lockStaleThreshold = options.lockStaleThreshold ?? 15000;
     this.liveOwnerStaleThreshold =
       options.liveOwnerStaleThreshold ?? Math.max(120000, this.lockStaleThreshold * 4);
+    this.acquireCeilingMs = options.acquireCeilingMs ?? LOCK_ACQUIRE_CEILING_MS;
     this.processLifecycle = options.processLifecycle ?? null;
   }
 
   /**
    * Decide whether a contended lock may be reclaimed.
-   *   - Dead owner            → reclaim immediately (crash cleanup).
-   *   - Legacy lock (no pid)  → reclaim once aged past the normal stale threshold.
-   *   - Provably-alive owner  → reclaim ONLY after liveOwnerStaleThreshold, so we
-   *                             never steal a lock a live process is actively
-   *                             holding during a (possibly slow) critical section.
-   *                             Stealing it would let two writers run the same
-   *                             read-modify-write and lose an update.
+   *   - Dead owner              → reclaim immediately (crash cleanup).
+   *   - Provably-alive owner    → reclaim ONLY after liveOwnerStaleThreshold, so we
+   *                               never steal a lock a live process is actively
+   *                               holding during a (possibly slow) critical section.
+   *                               Stealing it would let two writers run the same
+   *                               read-modify-write and lose an update.
+   *   - Unidentifiable (no pid) → judged by whether this lock has a HEARTBEAT to
+   *                               judge it by; see below.
+   *
+   * The unidentifiable case used to take the short threshold outright, which put it
+   * on the crash-cleanup schedule on the strength of no evidence at all — while
+   * `_isOwnerAlive(null)` returns true, so the two halves openly disagreed. The
+   * forensic cost was the worse half: the reclaim log read
+   * "Cleaning up stale lock file (90s old, owner alive: true)", a sentence
+   * describing the one thing this class exists not to do.
+   *
+   * Treating it as fully alive instead was an overcorrection, and it broke a real
+   * recovery: a lock file torn by a SIGKILL between `open(...,'wx')` and `write()`
+   * is unidentifiable by construction, and holding it for the live-owner threshold
+   * made knowledge-store init FAIL where it used to reclaim and proceed.
+   *
+   * The resolution is that a pid is not the only liveness evidence available. A
+   * lock class with a heartbeat has its mtime refreshed by its holder, so an
+   * unreadable lock that has ALSO gone motionless past the stale threshold is very
+   * likely abandoned — the heartbeat is exactly the proof that does not need a pid.
+   * Where there is no heartbeat (the run semaphore opts out of live-owner reclaim,
+   * so its slot files never move) age proves nothing, and only the ten-minute
+   * ceiling keeps an unreadable slot file from holding a run slot for the life of
+   * the machine.
    */
   private async _shouldReclaim(
     pid: number | null,
@@ -148,10 +222,49 @@ export class FileLockService implements IService {
     memo?: Map<string, boolean>,
   ): Promise<boolean> {
     if (!(await this._resolveOwnerAlive(pid, startTime, memo))) return true;
-    if (pid === null) return lockAge > this.lockStaleThreshold;
+    if (pid === null) return lockAge > this._unidentifiedStaleThreshold();
     return lockAge > this.liveOwnerStaleThreshold;
   }
 
+  /**
+   * How long an unidentifiable lock is left alone before it may be reclaimed.
+   *
+   * With a heartbeat running for this lock class, a motionless file is evidence of
+   * abandonment even though its owner cannot be named, so the ordinary stale
+   * threshold applies. Without one — `_startHeartbeat` skips the effectively
+   * infinite opt-out — age carries no information and only the absolute ceiling
+   * prevents an unreadable file from holding the lock forever.
+   */
+  private _unidentifiedStaleThreshold(): number {
+    const heartbeatInterval = this._heartbeatIntervalMs();
+    // Never shorter than a heartbeat interval plus a tick, or a live holder could be
+    // reclaimed in the gap between two refreshes.
+    return heartbeatInterval === null
+      ? UNIDENTIFIED_MAX_STALE_MS
+      : Math.max(this.lockStaleThreshold, heartbeatInterval * 2);
+  }
+
+  /**
+   * The interval at which a held lock's mtime is refreshed, or null when this lock
+   * class opts out of live-owner reclaim entirely and therefore runs no heartbeat.
+   *
+   * One helper because two call sites depend on the SAME number: the timer that does
+   * the refreshing, and the window that judges an unrefreshed lock abandoned. They
+   * were computed independently from the same formula, which is a drift waiting to
+   * happen in exactly the direction that reclaims a live holder.
+   *
+   * The opt-out is detected from the DERIVED interval, before the cap: an effectively
+   * infinite liveOwnerStaleThreshold is how the run semaphore says "never steal from a
+   * live owner", and clamping it into a runnable timer would quietly start a heartbeat
+   * there — which in turn would shorten its unidentified window from the ten-minute
+   * ceiling to seconds, reclaiming run slots the opt-out exists to protect.
+   */
+  private _heartbeatIntervalMs(): number | null {
+    const derived = Math.ceil(this.liveOwnerStaleThreshold / 4);
+    if (derived > MAX_TIMER_DELAY_MS) return null;
+    return Math.min(HEARTBEAT_MAX_INTERVAL_MS, Math.max(HEARTBEAT_MIN_INTERVAL_MS, derived));
+  }
+
   /**
    * Liveness of a contended owner, memoized per acquire() episode. The memo (keyed
    * by pid:startTime) ensures the PID+start-time check — which may spawn a ps/powershell
@@ -292,12 +405,12 @@ export class FileLockService implements IService {
    */
   private _startHeartbeat(): void {
     this._stopHeartbeat();
-    const interval = Math.max(HEARTBEAT_MIN_INTERVAL_MS, Math.ceil(this.liveOwnerStaleThreshold / 4));
     // An effectively-infinite threshold (the run-semaphore's never-steal-from-a-
     // live-owner opt-out) needs no heartbeat — and would overflow the 32-bit
-    // timer delay, which Node clamps to 1ms (a utimes busy-loop). Skip entirely;
-    // the opt-out's semantics are unchanged.
-    if (interval > MAX_TIMER_DELAY_MS) return;
+    // timer delay, which Node clamps to 1ms (a utimes busy-loop). _heartbeatIntervalMs
+    // returns null there; the opt-out's semantics are unchanged.
+    const interval = this._heartbeatIntervalMs();
+    if (interval === null) return;
     this.heartbeatTimer = setInterval(() => {
       const handle = this.lockHandle;
       if (handle === null) return;
@@ -370,8 +483,16 @@ export class FileLockService implements IService {
       const ownerAlive = await this._resolveOwnerAlive(parsed?.pid ?? null, parsed?.startTime ?? null, memo);
 
       if (await this._shouldReclaim(parsed?.pid ?? null, parsed?.startTime ?? null, lockAge, memo)) {
+        // Name the lock and say what was actually known about its owner. The old
+        // line printed "owner alive: true" for a lock with no readable pid —
+        // because unknown owners report as alive — so the reclaim read as a steal
+        // from a living process, and it named no path, in a log every concurrent
+        // process shares.
+        const owner = parsed?.pid == null
+          ? 'owner unidentifiable (unreadable lock content)'
+          : `owner pid ${parsed.pid} ${ownerAlive ? 'alive but silent' : 'gone'}`;
         logger.log(
-          `[FileLockService] Cleaning up stale lock file (${Math.round(lockAge / 1000)}s old, owner alive: ${ownerAlive})`
+          `[FileLockService] Reclaiming ${this.lockName} — untouched for ${Math.round(lockAge / 1000)}s, ${owner}`
         );
         // The liveness lookups above can take long enough for another process to
         // reclaim the stale lock and write its own fresh one at this path. A bare
@@ -391,7 +512,7 @@ export class FileLockService implements IService {
             return;
           }
           await fs.unlink(trashPath).catch(() => { /* leaked copy; the sweep above collects it */ });
-          logger.log('[FileLockService] Stale lock removed');
+          logger.log(`[FileLockService] Reclaimed ${this.lockName}`);
         } catch {
           // A failure AFTER the rename means the verify never ran — restore
           // rather than leave the (possibly live) lock destroyed in the trash.
@@ -469,14 +590,39 @@ export class FileLockService implements IService {
     this.resolveTurn = myResolve;
 
     const startTime = Date.now();
+    // Last moment the contended lock demonstrably moved: it changed hands, or its
+    // holder refreshed the mtime its heartbeat maintains. `lockTimeout` is measured
+    // from here rather than from startTime, so waiting behind work in progress is
+    // free and only a genuinely motionless lock times out.
+    let lastProgressAt = startTime;
+    let lastSeenUuid: string | null = null;
+    let lastSeenMtime = 0;
     let contentionCount = 0;
     // One entry per distinct (pid,startTime) owner seen during THIS acquireLock()
     // call, so a live contended owner waited out across many ~100ms retry ticks is
     // liveness-checked once, not once per tick (no ps/powershell storm on non-Linux).
     const contendedOwnerAliveMemo = new Map<string, boolean>();
 
+    // `lockRetries` was sized to expire at roughly the same moment as the old
+    // elapsed-time timeout (200 x 100ms = 20s), so leaving it alone would cap the
+    // progress-aware wait right back at the bound it replaces. A try-take
+    // (lockTimeout 0) keeps its configured count exactly: the run semaphore sizes
+    // it to complete a dead-owner reclaim and reacquire, and must still fail fast.
+    const maxAttempts = this.lockTimeout === 0
+      ? this.lockRetries
+      : this.lockRetries + Math.ceil(this.acquireCeilingMs / Math.max(1, this.lockRetryDelay));
+
     try {
-      for (let _attempt = 0; _attempt < this.lockRetries; _attempt++) {
+      for (let _attempt = 0; _attempt < maxAttempts; _attempt++) {
+        // Bound EVERY iteration, not just the ones that reach the timeout check
+        // below: the reclaim branches `continue` without sleeping, so a lock that
+        // kept changing hands could otherwise spin here indefinitely.
+        if (this.lockTimeout > 0 && Date.now() - startTime >= this.acquireCeilingMs) {
+          metrics.increment('state_lock_acquire_total', 1, { status: 'timeout' });
+          throw new Error(
+            `Failed to acquire lock at ${this.lockFilePath}: gave up after ${Date.now() - startTime}ms of contention (ceiling ${this.acquireCeilingMs}ms)`,
+          );
+        }
         // cleanup() can retire the instance while we are contending here — its
         // drain polls only the HELD handle, so a caller mid-retry (holding
         // nothing yet) is invisible to it and would otherwise win the lock and
@@ -548,6 +694,12 @@ export class FileLockService implements IService {
             const errnoError = error as NodeJS.ErrnoException;
             if (errnoError.code === 'EEXIST') {
               contentionCount++;
+              // When the contended lock becomes eligible for reclaim, if this tick can
+              // know it. Scoped per observation: a tick that could not read the lock
+              // learned nothing, and an eligibility carried over from an earlier one is
+              // no longer evidence of anything. See the stall check below for why the
+              // stall bound must not fire before this moment.
+              let reclaimEligibleAt: number | null = null;
               // Read lock content to verify ownership and liveness before considering stale
               try {
                 const rawContent = await fs.readFile(this.lockFilePath, 'utf-8');
@@ -556,6 +708,30 @@ export class FileLockService implements IService {
                 const stats = await fs.stat(this.lockFilePath);
                 const lockAge = Date.now() - stats.mtimeMs;
 
+                // Progress: a different owner than last time, or the same owner's
+                // heartbeat freshening the file. Either way the system is moving and
+                // this wait is queueing, not hanging.
+                if (lastSeenUuid !== null && (lockUuid !== lastSeenUuid || stats.mtimeMs > lastSeenMtime)) {
+                    lastProgressAt = Date.now();
+                }
+                lastSeenUuid = lockUuid;
+                lastSeenMtime = stats.mtimeMs;
+
+                // A lock with no readable pid is judged purely on age (see
+                // _shouldReclaim), and its mtime never moves again — so it can never
+                // show progress and the stall bound is guaranteed to expire first.
+                // Record when it becomes reclaimable so the wait below can outlast the
+                // stall bound rather than abandoning the recovery at 20s.
+                //
+                // Deliberately NOT extended to a live owner: there the heartbeat is the
+                // designed progress signal, its absence past lockTimeout is real evidence
+                // of a wedge, and reporting that to the caller promptly beats waiting out
+                // a two-to-four-minute live-owner threshold on a lock we may well never
+                // be handed.
+                reclaimEligibleAt = (parsed?.pid ?? null) === null
+                  ? stats.mtimeMs + this._unidentifiedStaleThreshold()
+                  : null;
+
                 if (lockUuid === this.lockUuid) {
                   // This is our own lock — a previous attempt was interrupted. Reclaim.
                   try { await fs.unlink(this.lockFilePath); } catch { /* ignore */ }
@@ -613,12 +789,31 @@ export class FileLockService implements IService {
                 // Fall through to the timeout check + backoff below.
               }
 
-              if (Date.now() - startTime >= this.lockTimeout) {
-                const duration = Date.now() - startTime;
-                metrics.observe('state_lock_acquire_duration_ms', duration, { status: 'timeout' });
+              const stalledFor = Date.now() - lastProgressAt;
+              const waitedFor = Date.now() - startTime;
+              // Never give up on a lock we are about to be ALLOWED to take. The stall
+              // bound and the reclaim window are two different clocks, and on every
+              // lock class configured here the stall bound is the shorter one — so a
+              // torn, ownerless lock timed the acquirer out before its reclaim window
+              // opened, and the pid-less recovery path could not run. The knowledge-store
+              // init lock is the measured case: reclaimable at 60s, abandoned at 20s.
+              // The ceiling below is still absolute.
+              // A try-take (lockTimeout 0) is excluded outright: it is configured to fail
+              // on the spot rather than wait for anything, and the run semaphore's slot
+              // files carry a TEN-MINUTE unidentified window, so honouring eligibility
+              // there would turn "is this slot free right now" into a ten-minute block.
+              const awaitingReclaim =
+                this.lockTimeout > 0 && reclaimEligibleAt !== null && Date.now() < reclaimEligibleAt;
+              if ((stalledFor >= this.lockTimeout && !awaitingReclaim) || waitedFor >= this.acquireCeilingMs) {
+                metrics.observe('state_lock_acquire_duration_ms', waitedFor, { status: 'timeout' });
                 metrics.increment('state_lock_acquire_total', 1, { status: 'timeout' });
+                // Say which bound was hit. "Timeout after 20000ms" on a lock whose
+                // holder was working the whole time sent readers looking for a
+                // deadlock that was never there.
                 throw new Error(
-                  `Failed to acquire lock at ${this.lockFilePath}: timeout after ${this.lockTimeout}ms`,
+                  waitedFor >= this.acquireCeilingMs
+                    ? `Failed to acquire lock at ${this.lockFilePath}: gave up after ${waitedFor}ms of contention (ceiling ${this.acquireCeilingMs}ms)`
+                    : `Failed to acquire lock at ${this.lockFilePath}: timeout after ${this.lockTimeout}ms with no sign of progress from its holder`,
                   { cause: error },
                 );
               }

package/src/infrastructure/knowledge-store-service.ts

@@ -193,6 +193,20 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
           this._initLock = new FileLockService({
             lockFilePath: lockPath,
             lockStaleThreshold: 60000,
+            // The default lockTimeout (20s, class default) is the WAITING side's own
+            // "give up and throw" bound — separate from, and much shorter than, the
+            // liveOwnerStaleThreshold (120s default) that actually governs when a
+            // provably-alive holder's lock may be physically reclaimed. This lock
+            // protects a post-run FTS rebuild plus optimize with no fixed duration; if
+            // that holder has a stretch of non-yielding synchronous work (a single
+            // large better-sqlite3 call blocks the event loop for its full duration,
+            // during which the heartbeat literally cannot fire) longer than 20s, a
+            // waiting caller would abandon and throw "no sign of progress from its
+            // holder" against a holder that was never actually wedged — up to 100s
+            // before the lock would even become reclaim-eligible. Raised to match
+            // liveOwnerStaleThreshold's own default so a live holder is given the same
+            // grace period here that the reclaim logic itself already grants it.
+            lockTimeout: 120000,
             processLifecycle,
           });
           await this._initLock.initialize();

package/src/infrastructure/research-run-semaphore.ts

@@ -184,7 +184,7 @@ export class ResearchRunSemaphore implements IService {
       lockTimeout: 0, // fail immediately on a live-contended slot (try-take)
       lockRetries: 24, // enough iterations to complete a dead-owner reclaim + reacquire
       lockRetryDelay: 50,
-      lockStaleThreshold: 15000, // legacy (no-pid) locks: reclaim once 15s old
+      lockStaleThreshold: 15000, // derivation base only; no-pid locks now follow the live-owner rule under its 10-minute ceiling
       // A live process holding a run slot for the entire (minutes-long) run is the
       // NORMAL case, not a stall. Never steal from a live owner — doing so would
       // admit an (N+1)th concurrent run, the exact condition this guard prevents.

package/src/infrastructure/service-initialization.ts

@@ -65,6 +65,13 @@ export function registerInfrastructureServices(container: ServiceContainer = get
       return new FileLockService({
         lockFilePath: pathConfig.getLockFilePath(),
         processLifecycle,
+        // The shared state lock guards a temp-file write plus fsync plus rename —
+        // sub-second work, unlike the knowledge-store init lock that the default
+        // five-minute ceiling was sized for (an FTS rebuild has no fixed duration).
+        // Inheriting that default here would let SDK teardown, which awaits a state
+        // write, stall for five minutes under sustained contention where it used to
+        // fail in twenty seconds. A ceiling should match what its lock protects.
+        acquireCeilingMs: 30_000,
       });
     },
     {

package/src/knowledge/embedder-init.ts

@@ -73,14 +73,31 @@ export async function warmupPipeline(
   useCache: boolean
 ): Promise<{ dummy: any; success: boolean; error?: Error }> {
   try {
-    const dummy = await withTimeout(
-      pipeline('warmup', { 
-        pooling: poolingMode as any, 
-        normalize: false,
-        ...(useCache === false ? { use_cache: false } : {}),
-      }),
-      20_000,
-      'Model warmup timed out after 20000ms.'
+    // Inside runCapturingStderr for the same reason the load above is, and for one
+    // more: warmup is the FIRST inference, so it is where a GPU that cannot serve the
+    // model actually fails. transformers.js reports that failure by printing
+    // "An error occurred during model execution: ..." plus a dump of the input tensors
+    // to console.error BEFORE rejecting — and this call was the only pipeline
+    // invocation in the module left unwrapped, so that dump went straight to the
+    // user's terminal. It is pure noise there: the rejection is caught two frames up,
+    // the embedder falls back to CPU, and the run continues to completion. What the
+    // user saw was a multi-line raw error for something that had already been handled.
+    //
+    // The load's capture ends before this runs, so wrapping the caller is not enough;
+    // and the server's captureStdio around startServer() covers only the FIRST
+    // initialization, not the re-initialization that follows every idle-timeout
+    // teardown — which is exactly when this was observed (2026-08-16, the fourth
+    // init of the session).
+    const dummy = await logger.runCapturingStderr(async () =>
+      withTimeout(
+        pipeline('warmup', {
+          pooling: poolingMode as any,
+          normalize: false,
+          ...(useCache === false ? { use_cache: false } : {}),
+        }),
+        20_000,
+        'Model warmup timed out after 20000ms.'
+      ),
     );
     return { dummy, success: true };
   } catch (warmupErr) {
@@ -402,16 +419,25 @@ export async function handleWebGPUWarmupError(
   initializationTimeoutMs: number,
   useCache: boolean
 ): Promise<{ success: boolean; pipeline?: FeatureExtractionPipeline; error?: Error; dummy?: any }> {
-  const isValidationError = warmupErr.message.toLowerCase().includes('validation');
-  
   markWebGpuFallback();
-  
-  if (isValidationError) {
-    logger.warn('[embedder] WebGPU validation error during warmup — falling back to CPU');
-    logger.debug('[embedder] Validation error details:', warmupErr.message);
-  } else {
-    logger.warn('[embedder] WebGPU OOM during warmup — falling back to CPU');
-  }
+
+  // Deliberately does NOT try to name the cause from this error's text. The split it
+  // replaced ("validation error" vs "OOM", chosen on whether the message contains
+  // "validation") reported the SYMPTOM as the diagnosis, and got it backwards in the
+  // case that matters: when the GPU is out of VRAM, Dawn raises
+  // VK_ERROR_OUT_OF_DEVICE_MEMORY through its device-error callback — native stderr,
+  // not this promise — and CreateBuffer then hands back an invalid buffer. What
+  // reaches JS is the first thing to *use* that buffer, i.e.
+  // "[Invalid Buffer] ... While calling CreateBindGroup", which contains the word
+  // "validation" and so was logged as a validation error. A 2026-08-16 run read
+  // exactly that way in the log while eighteen vkAllocateMemory failures sat directly
+  // above it. The native lines are captured into this same log, so point at them
+  // rather than paraphrasing them wrongly.
+  logger.warn(
+    '[embedder] WebGPU error during warmup — falling back to CPU. ' +
+    'The underlying cause (commonly VRAM exhaustion) is in the native Dawn/ONNX output logged just above this line.',
+  );
+  logger.debug('[embedder] Warmup error details:', warmupErr.message);
   
   // Clean up
   if (pipeline) {

package/src/logger.ts

@@ -259,6 +259,17 @@ export class Logger implements ILogger {
       timestamp,
       level,
       ...getLogContext(),
+      // Every process writes to the SAME log file — up to three concurrent research
+      // runs plus the elected browser-pool leader — and until this field existed
+      // there was nothing to tell their lines apart. Reconstructing a lock
+      // contention from the log then meant inferring process boundaries from
+      // timing, which is guesswork: two runs a second apart interleave into what
+      // reads like one process contradicting itself. Twelve bytes a line buys
+      // attributable forensics.
+      //
+      // After the context spread, not before: which process wrote a line is not
+      // something a scoped context gets to claim otherwise.
+      pid: process.pid,
       message,
       ...(firstError
         ? { errorMessage: redactSecrets(firstError.message), errorStack: redactSecrets(firstError.stack ?? '') }