@lincoln504/pi-research 1.3.12 → 1.4.3

package/src/infrastructure/file-lock-service.ts

@@ -44,6 +44,49 @@ const TRASH_SWEEP_AGE_MS = 10 * 60_000;
  */
 const HEARTBEAT_MIN_INTERVAL_MS = 2_000;
 
+/**
+ * Ceiling on the held-lock mtime heartbeat interval.
+ *
+ * The interval derives from liveOwnerStaleThreshold/4, which is sized so a live
+ * holder is never judged stale between two refreshes. That is the right shape for
+ * the live-owner rule and the wrong size for everything downstream of it: the
+ * knowledge-store init lock derived a 60s interval, and _unidentifiedStaleThreshold
+ * doubles it, so a lock torn by SIGKILL — unidentifiable by construction — was left
+ * alone for two minutes while the acquirer's stall bound gave up after twenty
+ * seconds. The recovery the unidentified rule exists to allow could not happen.
+ *
+ * A utimes every few seconds costs nothing, so cap it. The cap does not change who
+ * may be reclaimed: liveOwnerStaleThreshold is untouched, and refreshing more often
+ * than required only makes a live holder harder to mistake for a dead one.
+ */
+const HEARTBEAT_MAX_INTERVAL_MS = 5_000;
+
+/**
+ * Ceiling on how long a lock whose owner cannot be identified is left alone.
+ *
+ * Such a lock is treated as live (we could not prove otherwise), but "treated as
+ * live" must not mean "immortal" — see _shouldReclaim for why the never-steal
+ * opt-out would otherwise strand a run slot permanently.
+ */
+const UNIDENTIFIED_MAX_STALE_MS = 10 * 60_000;
+
+/**
+ * Absolute ceiling on one acquireLock() wait, however healthy the holder looks.
+ *
+ * `lockTimeout` bounds a STALL — time in which the lock did not change hands and
+ * its holder did not refresh it. It deliberately does not bound a wait behind a
+ * holder that is visibly working, because that turned ordinary contention into
+ * failed runs: the knowledge-store init lock allowed 20s while the same lock
+ * covers a post-run FTS rebuild plus optimize, so a second run starting during
+ * another's housekeeping failed outright. Twice, on 2026-08-14. Widening the
+ * constant is the fix that does not work — housekeeping has no fixed duration.
+ *
+ * This ceiling is what keeps "wait while progress is happening" from meaning
+ * "wait forever" under sustained contention, where the lock keeps changing hands
+ * and every observation looks like progress while this caller never wins.
+ */
+export const LOCK_ACQUIRE_CEILING_MS = 5 * 60_000;
+
 /** setTimeout/setInterval clamp delays above 2^31-1 to 1ms — treat anything
  *  bigger as "no heartbeat" rather than a busy-loop. */
 const MAX_TIMER_DELAY_MS = 0x7fffffff;
@@ -62,6 +105,13 @@ export interface FileLockOptions {
   lockRetryDelay?: number;
   /** Stale lock threshold in milliseconds (default: 15 seconds) */
   lockStaleThreshold?: number;
+  /**
+   * Absolute ceiling on one acquireLock() wait, however healthy the holder looks
+   * (default: LOCK_ACQUIRE_CEILING_MS). `lockTimeout` bounds a stall; this bounds
+   * the total. Exposed so the bound is reachable in a test — a five-minute
+   * backstop that no test can drive is a five-minute backstop nobody has checked.
+   */
+  acquireCeilingMs?: number;
   /**
    * Stale threshold (ms) applied ONLY when the lock's owner process is provably
    * alive. Much larger than lockStaleThreshold so a live process holding the lock
@@ -95,6 +145,7 @@ export class FileLockService implements IService {
   private readonly lockRetryDelay: number;
   private readonly lockStaleThreshold: number;
   private readonly liveOwnerStaleThreshold: number;
+  private readonly acquireCeilingMs: number;
   private readonly processLifecycle: IProcessLifecycle | null;
 
   // Lock tracking
@@ -128,18 +179,41 @@ export class FileLockService implements IService {
     this.lockStaleThreshold = options.lockStaleThreshold ?? 15000;
     this.liveOwnerStaleThreshold =
       options.liveOwnerStaleThreshold ?? Math.max(120000, this.lockStaleThreshold * 4);
+    this.acquireCeilingMs = options.acquireCeilingMs ?? LOCK_ACQUIRE_CEILING_MS;
     this.processLifecycle = options.processLifecycle ?? null;
   }
 
   /**
    * Decide whether a contended lock may be reclaimed.
-   *   - Dead owner            → reclaim immediately (crash cleanup).
-   *   - Legacy lock (no pid)  → reclaim once aged past the normal stale threshold.
-   *   - Provably-alive owner  → reclaim ONLY after liveOwnerStaleThreshold, so we
-   *                             never steal a lock a live process is actively
-   *                             holding during a (possibly slow) critical section.
-   *                             Stealing it would let two writers run the same
-   *                             read-modify-write and lose an update.
+   *   - Dead owner              → reclaim immediately (crash cleanup).
+   *   - Provably-alive owner    → reclaim ONLY after liveOwnerStaleThreshold, so we
+   *                               never steal a lock a live process is actively
+   *                               holding during a (possibly slow) critical section.
+   *                               Stealing it would let two writers run the same
+   *                               read-modify-write and lose an update.
+   *   - Unidentifiable (no pid) → judged by whether this lock has a HEARTBEAT to
+   *                               judge it by; see below.
+   *
+   * The unidentifiable case used to take the short threshold outright, which put it
+   * on the crash-cleanup schedule on the strength of no evidence at all — while
+   * `_isOwnerAlive(null)` returns true, so the two halves openly disagreed. The
+   * forensic cost was the worse half: the reclaim log read
+   * "Cleaning up stale lock file (90s old, owner alive: true)", a sentence
+   * describing the one thing this class exists not to do.
+   *
+   * Treating it as fully alive instead was an overcorrection, and it broke a real
+   * recovery: a lock file torn by a SIGKILL between `open(...,'wx')` and `write()`
+   * is unidentifiable by construction, and holding it for the live-owner threshold
+   * made knowledge-store init FAIL where it used to reclaim and proceed.
+   *
+   * The resolution is that a pid is not the only liveness evidence available. A
+   * lock class with a heartbeat has its mtime refreshed by its holder, so an
+   * unreadable lock that has ALSO gone motionless past the stale threshold is very
+   * likely abandoned — the heartbeat is exactly the proof that does not need a pid.
+   * Where there is no heartbeat (the run semaphore opts out of live-owner reclaim,
+   * so its slot files never move) age proves nothing, and only the ten-minute
+   * ceiling keeps an unreadable slot file from holding a run slot for the life of
+   * the machine.
    */
   private async _shouldReclaim(
     pid: number | null,
@@ -148,10 +222,49 @@ export class FileLockService implements IService {
     memo?: Map<string, boolean>,
   ): Promise<boolean> {
     if (!(await this._resolveOwnerAlive(pid, startTime, memo))) return true;
-    if (pid === null) return lockAge > this.lockStaleThreshold;
+    if (pid === null) return lockAge > this._unidentifiedStaleThreshold();
     return lockAge > this.liveOwnerStaleThreshold;
   }
 
+  /**
+   * How long an unidentifiable lock is left alone before it may be reclaimed.
+   *
+   * With a heartbeat running for this lock class, a motionless file is evidence of
+   * abandonment even though its owner cannot be named, so the ordinary stale
+   * threshold applies. Without one — `_startHeartbeat` skips the effectively
+   * infinite opt-out — age carries no information and only the absolute ceiling
+   * prevents an unreadable file from holding the lock forever.
+   */
+  private _unidentifiedStaleThreshold(): number {
+    const heartbeatInterval = this._heartbeatIntervalMs();
+    // Never shorter than a heartbeat interval plus a tick, or a live holder could be
+    // reclaimed in the gap between two refreshes.
+    return heartbeatInterval === null
+      ? UNIDENTIFIED_MAX_STALE_MS
+      : Math.max(this.lockStaleThreshold, heartbeatInterval * 2);
+  }
+
+  /**
+   * The interval at which a held lock's mtime is refreshed, or null when this lock
+   * class opts out of live-owner reclaim entirely and therefore runs no heartbeat.
+   *
+   * One helper because two call sites depend on the SAME number: the timer that does
+   * the refreshing, and the window that judges an unrefreshed lock abandoned. They
+   * were computed independently from the same formula, which is a drift waiting to
+   * happen in exactly the direction that reclaims a live holder.
+   *
+   * The opt-out is detected from the DERIVED interval, before the cap: an effectively
+   * infinite liveOwnerStaleThreshold is how the run semaphore says "never steal from a
+   * live owner", and clamping it into a runnable timer would quietly start a heartbeat
+   * there — which in turn would shorten its unidentified window from the ten-minute
+   * ceiling to seconds, reclaiming run slots the opt-out exists to protect.
+   */
+  private _heartbeatIntervalMs(): number | null {
+    const derived = Math.ceil(this.liveOwnerStaleThreshold / 4);
+    if (derived > MAX_TIMER_DELAY_MS) return null;
+    return Math.min(HEARTBEAT_MAX_INTERVAL_MS, Math.max(HEARTBEAT_MIN_INTERVAL_MS, derived));
+  }
+
   /**
    * Liveness of a contended owner, memoized per acquire() episode. The memo (keyed
    * by pid:startTime) ensures the PID+start-time check — which may spawn a ps/powershell
@@ -292,12 +405,12 @@ export class FileLockService implements IService {
    */
   private _startHeartbeat(): void {
     this._stopHeartbeat();
-    const interval = Math.max(HEARTBEAT_MIN_INTERVAL_MS, Math.ceil(this.liveOwnerStaleThreshold / 4));
     // An effectively-infinite threshold (the run-semaphore's never-steal-from-a-
     // live-owner opt-out) needs no heartbeat — and would overflow the 32-bit
-    // timer delay, which Node clamps to 1ms (a utimes busy-loop). Skip entirely;
-    // the opt-out's semantics are unchanged.
-    if (interval > MAX_TIMER_DELAY_MS) return;
+    // timer delay, which Node clamps to 1ms (a utimes busy-loop). _heartbeatIntervalMs
+    // returns null there; the opt-out's semantics are unchanged.
+    const interval = this._heartbeatIntervalMs();
+    if (interval === null) return;
     this.heartbeatTimer = setInterval(() => {
       const handle = this.lockHandle;
       if (handle === null) return;
@@ -370,8 +483,16 @@ export class FileLockService implements IService {
       const ownerAlive = await this._resolveOwnerAlive(parsed?.pid ?? null, parsed?.startTime ?? null, memo);
 
       if (await this._shouldReclaim(parsed?.pid ?? null, parsed?.startTime ?? null, lockAge, memo)) {
+        // Name the lock and say what was actually known about its owner. The old
+        // line printed "owner alive: true" for a lock with no readable pid —
+        // because unknown owners report as alive — so the reclaim read as a steal
+        // from a living process, and it named no path, in a log every concurrent
+        // process shares.
+        const owner = parsed?.pid == null
+          ? 'owner unidentifiable (unreadable lock content)'
+          : `owner pid ${parsed.pid} ${ownerAlive ? 'alive but silent' : 'gone'}`;
         logger.log(
-          `[FileLockService] Cleaning up stale lock file (${Math.round(lockAge / 1000)}s old, owner alive: ${ownerAlive})`
+          `[FileLockService] Reclaiming ${this.lockName} — untouched for ${Math.round(lockAge / 1000)}s, ${owner}`
         );
         // The liveness lookups above can take long enough for another process to
         // reclaim the stale lock and write its own fresh one at this path. A bare
@@ -391,7 +512,7 @@ export class FileLockService implements IService {
             return;
           }
           await fs.unlink(trashPath).catch(() => { /* leaked copy; the sweep above collects it */ });
-          logger.log('[FileLockService] Stale lock removed');
+          logger.log(`[FileLockService] Reclaimed ${this.lockName}`);
         } catch {
           // A failure AFTER the rename means the verify never ran — restore
           // rather than leave the (possibly live) lock destroyed in the trash.
@@ -469,14 +590,39 @@ export class FileLockService implements IService {
     this.resolveTurn = myResolve;
 
     const startTime = Date.now();
+    // Last moment the contended lock demonstrably moved: it changed hands, or its
+    // holder refreshed the mtime its heartbeat maintains. `lockTimeout` is measured
+    // from here rather than from startTime, so waiting behind work in progress is
+    // free and only a genuinely motionless lock times out.
+    let lastProgressAt = startTime;
+    let lastSeenUuid: string | null = null;
+    let lastSeenMtime = 0;
     let contentionCount = 0;
     // One entry per distinct (pid,startTime) owner seen during THIS acquireLock()
     // call, so a live contended owner waited out across many ~100ms retry ticks is
     // liveness-checked once, not once per tick (no ps/powershell storm on non-Linux).
     const contendedOwnerAliveMemo = new Map<string, boolean>();
 
+    // `lockRetries` was sized to expire at roughly the same moment as the old
+    // elapsed-time timeout (200 x 100ms = 20s), so leaving it alone would cap the
+    // progress-aware wait right back at the bound it replaces. A try-take
+    // (lockTimeout 0) keeps its configured count exactly: the run semaphore sizes
+    // it to complete a dead-owner reclaim and reacquire, and must still fail fast.
+    const maxAttempts = this.lockTimeout === 0
+      ? this.lockRetries
+      : this.lockRetries + Math.ceil(this.acquireCeilingMs / Math.max(1, this.lockRetryDelay));
+
     try {
-      for (let _attempt = 0; _attempt < this.lockRetries; _attempt++) {
+      for (let _attempt = 0; _attempt < maxAttempts; _attempt++) {
+        // Bound EVERY iteration, not just the ones that reach the timeout check
+        // below: the reclaim branches `continue` without sleeping, so a lock that
+        // kept changing hands could otherwise spin here indefinitely.
+        if (this.lockTimeout > 0 && Date.now() - startTime >= this.acquireCeilingMs) {
+          metrics.increment('state_lock_acquire_total', 1, { status: 'timeout' });
+          throw new Error(
+            `Failed to acquire lock at ${this.lockFilePath}: gave up after ${Date.now() - startTime}ms of contention (ceiling ${this.acquireCeilingMs}ms)`,
+          );
+        }
         // cleanup() can retire the instance while we are contending here — its
         // drain polls only the HELD handle, so a caller mid-retry (holding
         // nothing yet) is invisible to it and would otherwise win the lock and
@@ -548,6 +694,12 @@ export class FileLockService implements IService {
             const errnoError = error as NodeJS.ErrnoException;
             if (errnoError.code === 'EEXIST') {
               contentionCount++;
+              // When the contended lock becomes eligible for reclaim, if this tick can
+              // know it. Scoped per observation: a tick that could not read the lock
+              // learned nothing, and an eligibility carried over from an earlier one is
+              // no longer evidence of anything. See the stall check below for why the
+              // stall bound must not fire before this moment.
+              let reclaimEligibleAt: number | null = null;
               // Read lock content to verify ownership and liveness before considering stale
               try {
                 const rawContent = await fs.readFile(this.lockFilePath, 'utf-8');
@@ -556,6 +708,30 @@ export class FileLockService implements IService {
                 const stats = await fs.stat(this.lockFilePath);
                 const lockAge = Date.now() - stats.mtimeMs;
 
+                // Progress: a different owner than last time, or the same owner's
+                // heartbeat freshening the file. Either way the system is moving and
+                // this wait is queueing, not hanging.
+                if (lastSeenUuid !== null && (lockUuid !== lastSeenUuid || stats.mtimeMs > lastSeenMtime)) {
+                    lastProgressAt = Date.now();
+                }
+                lastSeenUuid = lockUuid;
+                lastSeenMtime = stats.mtimeMs;
+
+                // A lock with no readable pid is judged purely on age (see
+                // _shouldReclaim), and its mtime never moves again — so it can never
+                // show progress and the stall bound is guaranteed to expire first.
+                // Record when it becomes reclaimable so the wait below can outlast the
+                // stall bound rather than abandoning the recovery at 20s.
+                //
+                // Deliberately NOT extended to a live owner: there the heartbeat is the
+                // designed progress signal, its absence past lockTimeout is real evidence
+                // of a wedge, and reporting that to the caller promptly beats waiting out
+                // a two-to-four-minute live-owner threshold on a lock we may well never
+                // be handed.
+                reclaimEligibleAt = (parsed?.pid ?? null) === null
+                  ? stats.mtimeMs + this._unidentifiedStaleThreshold()
+                  : null;
+
                 if (lockUuid === this.lockUuid) {
                   // This is our own lock — a previous attempt was interrupted. Reclaim.
                   try { await fs.unlink(this.lockFilePath); } catch { /* ignore */ }
@@ -613,12 +789,31 @@ export class FileLockService implements IService {
                 // Fall through to the timeout check + backoff below.
               }
 
-              if (Date.now() - startTime >= this.lockTimeout) {
-                const duration = Date.now() - startTime;
-                metrics.observe('state_lock_acquire_duration_ms', duration, { status: 'timeout' });
+              const stalledFor = Date.now() - lastProgressAt;
+              const waitedFor = Date.now() - startTime;
+              // Never give up on a lock we are about to be ALLOWED to take. The stall
+              // bound and the reclaim window are two different clocks, and on every
+              // lock class configured here the stall bound is the shorter one — so a
+              // torn, ownerless lock timed the acquirer out before its reclaim window
+              // opened, and the pid-less recovery path could not run. The knowledge-store
+              // init lock is the measured case: reclaimable at 60s, abandoned at 20s.
+              // The ceiling below is still absolute.
+              // A try-take (lockTimeout 0) is excluded outright: it is configured to fail
+              // on the spot rather than wait for anything, and the run semaphore's slot
+              // files carry a TEN-MINUTE unidentified window, so honouring eligibility
+              // there would turn "is this slot free right now" into a ten-minute block.
+              const awaitingReclaim =
+                this.lockTimeout > 0 && reclaimEligibleAt !== null && Date.now() < reclaimEligibleAt;
+              if ((stalledFor >= this.lockTimeout && !awaitingReclaim) || waitedFor >= this.acquireCeilingMs) {
+                metrics.observe('state_lock_acquire_duration_ms', waitedFor, { status: 'timeout' });
                 metrics.increment('state_lock_acquire_total', 1, { status: 'timeout' });
+                // Say which bound was hit. "Timeout after 20000ms" on a lock whose
+                // holder was working the whole time sent readers looking for a
+                // deadlock that was never there.
                 throw new Error(
-                  `Failed to acquire lock at ${this.lockFilePath}: timeout after ${this.lockTimeout}ms`,
+                  waitedFor >= this.acquireCeilingMs
+                    ? `Failed to acquire lock at ${this.lockFilePath}: gave up after ${waitedFor}ms of contention (ceiling ${this.acquireCeilingMs}ms)`
+                    : `Failed to acquire lock at ${this.lockFilePath}: timeout after ${this.lockTimeout}ms with no sign of progress from its holder`,
                   { cause: error },
                 );
               }

package/src/infrastructure/knowledge-store-service.ts

@@ -193,6 +193,20 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
           this._initLock = new FileLockService({
             lockFilePath: lockPath,
             lockStaleThreshold: 60000,
+            // The default lockTimeout (20s, class default) is the WAITING side's own
+            // "give up and throw" bound — separate from, and much shorter than, the
+            // liveOwnerStaleThreshold (120s default) that actually governs when a
+            // provably-alive holder's lock may be physically reclaimed. This lock
+            // protects a post-run FTS rebuild plus optimize with no fixed duration; if
+            // that holder has a stretch of non-yielding synchronous work (a single
+            // large better-sqlite3 call blocks the event loop for its full duration,
+            // during which the heartbeat literally cannot fire) longer than 20s, a
+            // waiting caller would abandon and throw "no sign of progress from its
+            // holder" against a holder that was never actually wedged — up to 100s
+            // before the lock would even become reclaim-eligible. Raised to match
+            // liveOwnerStaleThreshold's own default so a live holder is given the same
+            // grace period here that the reclaim logic itself already grants it.
+            lockTimeout: 120000,
             processLifecycle,
           });
           await this._initLock.initialize();

package/src/infrastructure/research-run-semaphore.ts

@@ -184,7 +184,7 @@ export class ResearchRunSemaphore implements IService {
       lockTimeout: 0, // fail immediately on a live-contended slot (try-take)
       lockRetries: 24, // enough iterations to complete a dead-owner reclaim + reacquire
       lockRetryDelay: 50,
-      lockStaleThreshold: 15000, // legacy (no-pid) locks: reclaim once 15s old
+      lockStaleThreshold: 15000, // derivation base only; no-pid locks now follow the live-owner rule under its 10-minute ceiling
       // A live process holding a run slot for the entire (minutes-long) run is the
       // NORMAL case, not a stall. Never steal from a live owner — doing so would
       // admit an (N+1)th concurrent run, the exact condition this guard prevents.

package/src/infrastructure/service-initialization.ts

@@ -65,6 +65,13 @@ export function registerInfrastructureServices(container: ServiceContainer = get
       return new FileLockService({
         lockFilePath: pathConfig.getLockFilePath(),
         processLifecycle,
+        // The shared state lock guards a temp-file write plus fsync plus rename —
+        // sub-second work, unlike the knowledge-store init lock that the default
+        // five-minute ceiling was sized for (an FTS rebuild has no fixed duration).
+        // Inheriting that default here would let SDK teardown, which awaits a state
+        // write, stall for five minutes under sustained contention where it used to
+        // fail in twenty seconds. A ceiling should match what its lock protects.
+        acquireCeilingMs: 30_000,
       });
     },
     {

package/src/knowledge/embedder-init.ts

@@ -73,14 +73,31 @@ export async function warmupPipeline(
   useCache: boolean
 ): Promise<{ dummy: any; success: boolean; error?: Error }> {
   try {
-    const dummy = await withTimeout(
-      pipeline('warmup', { 
-        pooling: poolingMode as any, 
-        normalize: false,
-        ...(useCache === false ? { use_cache: false } : {}),
-      }),
-      20_000,
-      'Model warmup timed out after 20000ms.'
+    // Inside runCapturingStderr for the same reason the load above is, and for one
+    // more: warmup is the FIRST inference, so it is where a GPU that cannot serve the
+    // model actually fails. transformers.js reports that failure by printing
+    // "An error occurred during model execution: ..." plus a dump of the input tensors
+    // to console.error BEFORE rejecting — and this call was the only pipeline
+    // invocation in the module left unwrapped, so that dump went straight to the
+    // user's terminal. It is pure noise there: the rejection is caught two frames up,
+    // the embedder falls back to CPU, and the run continues to completion. What the
+    // user saw was a multi-line raw error for something that had already been handled.
+    //
+    // The load's capture ends before this runs, so wrapping the caller is not enough;
+    // and the server's captureStdio around startServer() covers only the FIRST
+    // initialization, not the re-initialization that follows every idle-timeout
+    // teardown — which is exactly when this was observed (2026-08-16, the fourth
+    // init of the session).
+    const dummy = await logger.runCapturingStderr(async () =>
+      withTimeout(
+        pipeline('warmup', {
+          pooling: poolingMode as any,
+          normalize: false,
+          ...(useCache === false ? { use_cache: false } : {}),
+        }),
+        20_000,
+        'Model warmup timed out after 20000ms.'
+      ),
     );
     return { dummy, success: true };
   } catch (warmupErr) {
@@ -400,18 +417,28 @@ export async function handleWebGPUWarmupError(
   gpuLockHeld: boolean,
   model: string,
   initializationTimeoutMs: number,
-  useCache: boolean
+  useCache: boolean,
+  poolingMode: 'mean' | 'cls' | 'last_token' = 'mean'
 ): Promise<{ success: boolean; pipeline?: FeatureExtractionPipeline; error?: Error; dummy?: any }> {
-  const isValidationError = warmupErr.message.toLowerCase().includes('validation');
-  
   markWebGpuFallback();
-  
-  if (isValidationError) {
-    logger.warn('[embedder] WebGPU validation error during warmup — falling back to CPU');
-    logger.debug('[embedder] Validation error details:', warmupErr.message);
-  } else {
-    logger.warn('[embedder] WebGPU OOM during warmup — falling back to CPU');
-  }
+
+  // Deliberately does NOT try to name the cause from this error's text. The split it
+  // replaced ("validation error" vs "OOM", chosen on whether the message contains
+  // "validation") reported the SYMPTOM as the diagnosis, and got it backwards in the
+  // case that matters: when the GPU is out of VRAM, Dawn raises
+  // VK_ERROR_OUT_OF_DEVICE_MEMORY through its device-error callback — native stderr,
+  // not this promise — and CreateBuffer then hands back an invalid buffer. What
+  // reaches JS is the first thing to *use* that buffer, i.e.
+  // "[Invalid Buffer] ... While calling CreateBindGroup", which contains the word
+  // "validation" and so was logged as a validation error. A 2026-08-16 run read
+  // exactly that way in the log while eighteen vkAllocateMemory failures sat directly
+  // above it. The native lines are captured into this same log, so point at them
+  // rather than paraphrasing them wrongly.
+  logger.warn(
+    '[embedder] WebGPU error during warmup — falling back to CPU. ' +
+    'The underlying cause (commonly VRAM exhaustion) is in the native Dawn/ONNX output logged just above this line.',
+  );
+  logger.debug('[embedder] Warmup error details:', warmupErr.message);
   
   // Clean up
   if (pipeline) {
@@ -425,7 +452,7 @@ export async function handleWebGPUWarmupError(
     logger.info('[embedder] CPU pipeline loaded successfully');
     
     // Warmup CPU pipeline
-    const { dummy, success, error: cpuWarmupErr } = await warmupPipeline(cpuPipeline, 'mean', useCache);
+    const { dummy, success, error: cpuWarmupErr } = await warmupPipeline(cpuPipeline, poolingMode, useCache);
     if (!success || cpuWarmupErr) {
       logger.error('[embedder] CPU fallback pipeline warmup failed:', cpuWarmupErr);
       if (cpuPipeline) {

package/src/knowledge/embedder.ts

@@ -331,7 +331,8 @@ export class Embedder {
             this.gpuLockHeld,
             this.model,
             this.initializationTimeoutMs,
-            this.useCache
+            this.useCache,
+            this.poolingMode
           );
           if (!result.success) {
             throw result.error;
@@ -404,9 +405,16 @@ export class Embedder {
       // the VECTOR — the full text is still stored verbatim and covered by FTS — but the
       // truncation is otherwise silent, so surface it: a spike here means chunks are
       // arriving larger than the embed cap and vector recall on their tail is degraded.
+      let keep = maxChars;
+      // slice() counts UTF-16 code units, so the cut can land between the halves of
+      // a surrogate pair (emoji, rare CJK) — same guard as truncateWithMarker
+      // (utils/text-utils.ts): a lone high surrogate is invalid text some
+      // tokenizers/providers choke on. Back off one unit to restore well-formed text.
+      const lastKeptUnit = text.charCodeAt(keep - 1);
+      if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
       metrics.increment('embedder_truncations_total', 1, { model: this.model });
-      logger.debug(`[embedder] Truncated input for embedding: ${text.length} -> ${maxChars} chars (model=${this.model})`);
-      return text.slice(0, maxChars);
+      logger.debug(`[embedder] Truncated input for embedding: ${text.length} -> ${keep} chars (model=${this.model})`);
+      return text.slice(0, keep);
     }
     return text;
   }

package/src/knowledge/index.ts

@@ -1,4 +1,5 @@
 import * as fs from 'node:fs';
+import * as path from 'node:path';
 import { KnowledgeStore } from './store.ts';
 import { WriterQueue } from './writer-queue.ts';
 import { Chunker } from './chunker.ts';
@@ -108,19 +109,44 @@ export async function createKnowledgeStoreComponents(
 }
 
 /**
- * Force delete the entire knowledge store directory.
- * Used for system-level clearing or recovery from corruption.
+ * Force delete the ACTIVE table's own directory, to recover from a corrupted
+ * LanceDB manifest. Scoped to that one table, not the whole dbDir: dbDir is a
+ * single unified store shared by every workspace and the global scope (rows
+ * are scoped by a `workspace` column, not by separate directories), and it
+ * also holds `_backup_*`/`_migration_*` sibling directories — the deliberate,
+ * user-recoverable safety net the 'backup' migration strategy creates. A
+ * directory-wide wipe here previously destroyed every other project's data
+ * and every backup the instant ANY workspace's session hit a corrupted
+ * manifest, on the one code path whose entire purpose is recovering from
+ * exactly that kind of damage.
  */
 export async function forceDeleteKnowledgeStore(config?: Config, workspace?: string): Promise<void> {
   const dbDir = getDbDir(config, workspace);
-  if (fs.existsSync(dbDir)) {
-    try {
-      fs.rmSync(dbDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
-      fs.mkdirSync(dbDir, { recursive: true });
-      logger.info(`[knowledge] Knowledge store at ${dbDir} forcefully deleted and recreated.`);
-    } catch (err) {
-      logger.error(`[knowledge] Failed to forcefully delete directory ${dbDir}:`, err);
-      throw err;
+  if (!fs.existsSync(dbDir)) return;
+
+  // Mirrors KnowledgeStore's own manifest read (store.ts): the active table
+  // name defaults to 'knowledge' but can point elsewhere after a persisted
+  // rename failure. A missing or unreadable manifest is not itself a reason to
+  // widen the blast radius, so it falls back to the default rather than
+  // escalating to a dbDir-wide delete.
+  let tableName = 'knowledge';
+  const manifestPath = path.join(dbDir, 'store-manifest.json');
+  try {
+    if (fs.existsSync(manifestPath)) {
+      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as { activeTableName?: string };
+      if (manifest.activeTableName) tableName = manifest.activeTableName;
     }
+  } catch (err) {
+    logger.warn(`[knowledge] Failed to read manifest before forced delete, defaulting to table '${tableName}':`, err);
+  }
+
+  const tableDir = path.join(dbDir, `${tableName}.lance`);
+  if (!fs.existsSync(tableDir)) return;
+  try {
+    fs.rmSync(tableDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
+    logger.info(`[knowledge] Corrupted table directory ${tableDir} forcefully deleted; will be recreated on next init.`);
+  } catch (err) {
+    logger.error(`[knowledge] Failed to forcefully delete table directory ${tableDir}:`, err);
+    throw err;
   }
 }

package/src/knowledge/store.ts

@@ -502,12 +502,36 @@ export class KnowledgeStore implements IKnowledgeStore {
         // If canonical name was already something else and it still exists, 
         // we might need to handle it. But we just dropped it above.
         await fsPromises.rename(tempDir, canonicalDir);
-        
+
         this.tableName = canonicalName;
-        await this.saveManifest();
-        
-        // Reopen the canonical table so this.table reflects the renamed path
-        this.table = await this.db.openTable(this.tableName);
+        if (await this.saveManifest()) {
+          // Reopen the canonical table so this.table reflects the renamed path
+          this.table = await this.db.openTable(this.tableName);
+        } else {
+          // The directory rename to the canonical name already happened on disk,
+          // but the manifest still says tempTableName (persisted successfully
+          // above, before the old table was dropped). Left as-is, the next
+          // process start would read that stale name, find no such directory
+          // (renamed away), and silently CREATE A NEW EMPTY TABLE under it —
+          // stranding every re-embedded document in the now-unreferenced
+          // canonical directory with no recovery path (pruneOrphanedMigrationDirs
+          // doesn't recognize a bare 'knowledge.lance' as orphaned migration
+          // material). Revert the rename so the manifest (still correctly
+          // pointing at tempTableName, from the pre-drop save above) and the
+          // on-disk layout agree again — the data stays fully intact and
+          // accessible under its temp name either way. Deliberately NOT thrown:
+          // the migration itself fully succeeded and the in-memory handle is
+          // fine, so surfacing this as a migration failure would trip the
+          // caller's re-embed-failed fallback to the 'backup' strategy, which
+          // would needlessly rename this perfectly good, fully re-embedded data
+          // out of the way and start a fresh empty table over one manifest
+          // write hiccup.
+          logger.error('[store] Failed to persist canonical table name after rename — reverting to keep manifest and disk in sync.');
+          await fsPromises.rename(canonicalDir, tempDir);
+          this.tableName = tempTableName;
+          this.table = await this.db.openTable(this.tableName);
+          logger.info(`[store] Migrated data remains safe and accessible as ${tempTableName}; canonical rename deferred to a future successful manifest save.`);
+        }
       } catch (renameErr) {
         // If rename fails (cross-device, permissions, etc.) we keep the temp name
         // but CRITICALLY we now persist it in the manifest so future sessions

package/src/logger.ts

@@ -259,6 +259,17 @@ export class Logger implements ILogger {
       timestamp,
       level,
       ...getLogContext(),
+      // Every process writes to the SAME log file — up to three concurrent research
+      // runs plus the elected browser-pool leader — and until this field existed
+      // there was nothing to tell their lines apart. Reconstructing a lock
+      // contention from the log then meant inferring process boundaries from
+      // timing, which is guesswork: two runs a second apart interleave into what
+      // reads like one process contradicting itself. Twelve bytes a line buys
+      // attributable forensics.
+      //
+      // After the context spread, not before: which process wrote a line is not
+      // something a scoped context gets to claim otherwise.
+      pid: process.pid,
       message,
       ...(firstError
         ? { errorMessage: redactSecrets(firstError.message), errorStack: redactSecrets(firstError.stack ?? '') }