@lincoln504/pi-research 1.1.2 → 1.3.0

package/src/knowledge/embedder.ts

@@ -68,6 +68,36 @@ export class Embedder {
   private idleTimer: NodeJS.Timeout | null = null;
   private readonly IDLE_TIMEOUT_MS = 60 * 1000;
   private activeEmbeddings = 0;
+  /**
+   * Embed calls actively INSIDE the model (a pipeline invocation in flight), as
+   * opposed to activeEmbeddings, which counts whole embed()/embedMany() calls —
+   * including callers parked on recoveryPromise after a device loss. recoverToCpu
+   * drains THIS count before swapping the pipeline: parked callers still hold
+   * their activeEmbeddings slot (decremented downstream in their finally), so a
+   * drain on activeEmbeddings could never succeed with ≥2 concurrent device-loss
+   * failures and always burned the full timeout.
+   */
+  private activeInferences = 0;
+  /**
+   * Callers parked on recoveryPromise — they JOINED an in-flight recovery while
+   * still holding their activeEmbeddings slot (the decrement runs downstream in
+   * their finally, after the joined recovery settles). recoverToCpu's drain must
+   * distinguish them from callers still progressing toward the pipeline (parked
+   * on the per-call GPU lock, or between embedMany batches): parked joiners can
+   * never finish until recovery completes (waiting on them deadlocks into the
+   * drain timeout), while the progressing callers MUST be waited for — disposing
+   * the pipeline under them raises a TypeError no device-error classifier
+   * recognizes, so their embed fails unrecovered.
+   */
+  private recoveryWaiters = 0;
+  /**
+   * Why the in-flight dispose was started. An IDLE dispose is a memory-reclaim
+   * pause that a later embed is expected to undo, so initialize() waits it out and
+   * re-initializes. A TERMINAL dispose (shutdown, config change, explicit
+   * disposal) must never be undone — re-initializing there would resurrect the
+   * ONNX pipeline during teardown. Only meaningful while state === 'disposing'.
+   */
+  private disposeReason: 'idle' | 'terminal' = 'terminal';
 
   constructor(options: EmbedderOptions) {
     this.model = options.model;
@@ -113,7 +143,7 @@ export class Embedder {
       }
       if (this.state === 'ready') {
         logger.info(`[embedder] Idle timeout reached (${this.IDLE_TIMEOUT_MS}ms), releasing GPU memory...`);
-        this.dispose().catch(err => logger.warn('[embedder] Failed to dispose on idle:', err));
+        this.dispose('idle').catch(err => logger.warn('[embedder] Failed to dispose on idle:', err));
       }
     }, this.IDLE_TIMEOUT_MS);
     if (this.idleTimer) safeUnref(this.idleTimer);
@@ -129,7 +159,36 @@ export class Embedder {
   async initialize(): Promise<void> {
     if (this.state === 'ready') return;
     if (this.state === 'disposing') {
-      throw new Error('Cannot initialize while disposing');
+      // An IDLE dispose is a pause, not an end: the pipeline was released purely
+      // to reclaim GPU/host memory and the next embed is meant to bring it back.
+      // Throwing here lost that race — the idle timer fires, dispose() flips the
+      // state, and a document that arrives during the (up to ~5s) teardown got
+      // "Cannot initialize while disposing", which nothing classifies as
+      // transient, so the writer queue dropped it with no retry. Wait the dispose
+      // out and re-initialize instead.
+      if (this.disposeReason === 'idle') {
+        logger.debug('[embedder] initialize() raced an idle dispose — waiting for it to finish, then re-initializing.');
+        await this.disposePromise?.catch(() => { /* a failed idle dispose must not block the revive */ });
+        // Re-check: another caller may have revived (or terminally disposed) the
+        // embedder while we waited.
+        if ((this.state as EmbedderState) === 'ready') return;
+        // A terminal dispose landed as a DOWNGRADE onto the idle dispose we just
+        // awaited: state is 'idle', but reviving is forbidden — pre-fix this fell
+        // through to a full model load during process teardown. (A LATER, fresh
+        // initialize()/embed() after the completed dispose never enters this
+        // branch and stays revivable — that re-use is pinned behaviour.) The cast
+        // defeats control-flow narrowing, which cannot see the awaited dispose
+        // mutating the field — same idiom as the state re-checks around it.
+        if ((this.disposeReason as 'idle' | 'terminal') === 'terminal') {
+          throw new Error('Cannot initialize while disposing');
+        }
+        if ((this.state as EmbedderState) === 'disposing') {
+          throw new Error('Cannot initialize while disposing');
+        }
+      } else {
+        // Terminal dispose (shutdown / config change): must NOT be revived.
+        throw new Error('Cannot initialize while disposing');
+      }
     }
 
     if (this.initializingPromise) {
@@ -352,13 +411,36 @@ export class Embedder {
     return text;
   }
 
+  /**
+   * Run the pipeline over an input, tracked in activeInferences so recoverToCpu's
+   * drain loop waits only on calls genuinely inside the model.
+   */
+  private async runInference(input: string | string[]): Promise<any> {
+    // A disposal/recovery that raced this call may have nulled the pipeline out
+    // from under us. Calling through null raises a bare TypeError that nothing
+    // classifies (the writer queue drops the document as a generic ingest_error);
+    // throw the reconnect-classified signal instead — isEmbedderUnreachable
+    // matches "aborted" — so the caller retries via a fresh embedder.
+    if (!this.pipeline) {
+      throw new Error('[embedder] embed aborted: pipeline disposed before inference could run — reconnect required');
+    }
+    this.activeInferences++;
+    try {
+      return await logger.runCapturingStderr(async () => {
+        return await this.pipeline!(input, this.pipelineOpts());
+      });
+    } finally {
+      this.activeInferences--;
+    }
+  }
+
   async embed(text: string): Promise<Float32Array> {
     await this.initialize();
     // Compute the input BEFORE the active-count increment: truncateText is a pure
     // string op, and keeping it (and nothing else) ahead of the try/finally means
     // every path that can throw or await is inside the finally that decrements the
     // counter. Otherwise a throw here (or in acquireGpuLock) would leak the count,
-    // permanently wedging recoverToCpu (15s spin) and dispose (5s hang).
+    // permanently wedging dispose (5s hang).
     const input = this.truncateText(this.queryPrefix ? this.queryPrefix + text : text);
     this.stopIdleTimer();
     this.activeEmbeddings++;
@@ -371,9 +453,7 @@ export class Embedder {
           logger.warn('[embedder] GPU per-call lock timeout after 15s — proceeding without lock');
         }
       }
-      const output = await logger.runCapturingStderr(async () => {
-        return await this.pipeline!(input, this.pipelineOpts());
-      });
+      const output = await this.runInference(input);
       return output.data as Float32Array;
     } catch (err) {
       if (isWebGpuDeviceError(err)) {
@@ -388,9 +468,7 @@ export class Embedder {
         // dereferencing a null pipeline would raise a TypeError that MASKS the
         // original device error — rethrow the original instead.
         if (!this.pipeline) throw err;
-        const output = await logger.runCapturingStderr(async () => {
-          return await this.pipeline!(input, this.pipelineOpts());
-        });
+        const output = await this.runInference(input);
         return output.data as Float32Array;
       }
       throw err;
@@ -435,9 +513,7 @@ export class Embedder {
 
           let output: any;
           try {
-            output = await logger.runCapturingStderr(async () => {
-              return await this.pipeline!(batch, this.pipelineOpts());
-            });
+            output = await this.runInference(batch);
           } catch (err) {
             if (isWebGpuDeviceError(err)) {
               if (lockAcquired && this.stateManager) {
@@ -450,9 +526,7 @@ export class Embedder {
               // Same guard as embed(): recoverToCpu may have skipped (disposing/idle),
               // leaving pipeline null — rethrow the original error, not a TypeError.
               if (!this.pipeline) throw err;
-              output = await logger.runCapturingStderr(async () => {
-                return await this.pipeline!(batch, this.pipelineOpts());
-              });
+              output = await this.runInference(batch);
             } else {
               throw err;
             }
@@ -476,7 +550,17 @@ export class Embedder {
   }
 
   private async recoverToCpu(): Promise<void> {
-    if (this.recoveryPromise) return this.recoveryPromise;
+    if (this.recoveryPromise) {
+      // Counted join: this caller still holds its activeEmbeddings slot while
+      // parked here, and the drain below must not wait on parked joiners (see
+      // the recoveryWaiters field comment).
+      this.recoveryWaiters++;
+      try {
+        return await this.recoveryPromise;
+      } finally {
+        this.recoveryWaiters--;
+      }
+    }
 
     // Guard: if disposal has already started, skip recovery — the embedder is going away.
     if (this.state === 'disposing' || this.state === 'idle') {
@@ -484,22 +568,42 @@ export class Embedder {
       return;
     }
 
-    this.recoveryPromise = (async () => {
+    // Close the initialize() re-entry window SYNCHRONOUSLY, before the first
+    // await: initialize() only checks 'ready'/'disposing' and initializingPromise,
+    // so until the recovery promise below is published, a concurrent
+    // embed()/embedMany() passed every guard and launched a second concurrent
+    // _initializeInternal — racing pipeline/state assignments, leaking the losing
+    // pipeline, and (device was flipped to 'cpu' only after the drain loop)
+    // reloading the exact WebGPU backend that just failed. Flip device first so
+    // no joiner can ever observe 'webgpu' again this session.
+    this.state = 'initializing';
+    this.device = 'cpu';
+
+    const recovery = (async () => {
       logger.warn('[embedder] WebGPU device error detected during operation — falling back to CPU for the remainder of this session');
-      
+
       markWebGpuFallback();
-      
+
       await releaseGpuLock(this.stateManager, this.gpuLockHeld);
       this.gpuLockHeld = false;
-      
-      this.state = 'initializing';
-      
-      // Wait for other concurrent embeddings to finish before disposing the pipeline
+
+      // Drain in-flight callers before disposing the pipeline, on BOTH counts:
+      //  - activeInferences > 0: a call genuinely inside the model.
+      //  - activeEmbeddings - recoveryWaiters > 1: callers past initialize() but
+      //    not yet inside the model — parked on the per-call GPU lock, or between
+      //    embedMany batches. Self counts as 1 (this recovery runs from a caller
+      //    holding its own slot); parked joiners are subtracted because they can
+      //    never finish until THIS recovery completes (an uncorrected
+      //    activeEmbeddings drain always burned the full timeout under
+      //    multi-caller device loss). An inferences-only drain (the prior form)
+      //    disposed the pipeline under a lock-parked caller, whose embed then
+      //    failed with an unclassified TypeError, masking the device loss.
       const maxWaitMs = 15000;
       const startTime = Date.now();
-      // We expect activeEmbeddings to be at least 1 (the current caller).
-      // If multiple callers are in recoverToCpu, they all wait for recoveryPromise.
-      while (this.activeEmbeddings > 1 && (Date.now() - startTime) < maxWaitMs) {
+      while (
+        (this.activeInferences > 0 || this.activeEmbeddings - this.recoveryWaiters > 1) &&
+        (Date.now() - startTime) < maxWaitMs
+      ) {
         await new Promise(resolve => setTimeout(resolve, 50));
       }
 
@@ -507,48 +611,77 @@ export class Embedder {
         try { if (typeof (this.pipeline as any).dispose === 'function') await (this.pipeline as DisposablePipeline).dispose(); } catch (err) { logger.warn('[embedder] Error disposing pipeline:', err); }
         this.pipeline = null;
       }
-      this.device = 'cpu';
-      
-      this.initializingPromise = this._initializeInternal();
+
       try {
-          await this.initializingPromise;
-          this.state = 'ready';
+        await this._initializeInternal();
       } catch (e) {
-          this.state = 'failed';
-          throw e;
-      } finally {
-          this.initializingPromise = null;
+        this.state = 'failed';
+        throw e;
       }
+      // Same guard as initialize(): a terminal dispose may have landed while the
+      // CPU pipeline loaded — never overwrite 'disposing' with 'ready'. (Cast
+      // defeats control-flow narrowing, which cannot see the awaited init racing
+      // dispose() — same idiom as the state re-checks in initialize().)
+      if ((this.state as EmbedderState) === 'disposing') {
+        logger.warn('[embedder] Recovery finished but embedder was disposed in the meantime.');
+        return;
+      }
+      this.state = 'ready';
       logger.warn('[embedder] CPU fallback recovery complete.');
     })();
 
+    this.recoveryPromise = recovery;
+    // Publish for initialize(): a caller arriving mid-recovery must join the
+    // WHOLE recovery (drain + pipeline swap + re-init), not start its own. This
+    // and the assignments above run before recovery's first await, so there is
+    // no window in which a joiner can slip past.
+    this.initializingPromise = recovery;
+
     try {
-      await this.recoveryPromise;
+      await recovery;
     } finally {
-      this.recoveryPromise = null;
+      if (this.recoveryPromise === recovery) this.recoveryPromise = null;
+      // Guarded: by the time this finally runs, a fresh initialize() may already
+      // own initializingPromise (e.g. after a failed recovery) — never null it out
+      // from under that caller.
+      if (this.initializingPromise === recovery) this.initializingPromise = null;
     }
   }
 
-  async dispose(): Promise<void> {
+  /**
+   * @param reason `'idle'` for the memory-reclaim pause driven by the idle timer —
+   *   a later embed is allowed to revive the embedder. Anything else is terminal
+   *   (shutdown, config change, explicit disposal) and must never be revived, so
+   *   the default is deliberately the safe one: callers that do not opt in get
+   *   terminal semantics.
+   */
+  async dispose(reason: 'idle' | 'terminal' = 'terminal'): Promise<void> {
     if (this.state === 'idle') return;
     this.stopIdleTimer();
 
     if (this.state === 'disposing' && this.disposePromise) {
+      // A terminal dispose landing on top of an in-flight idle dispose must
+      // downgrade the reason, so a concurrent initialize() cannot revive what is
+      // now a shutdown. Never the reverse.
+      if (reason === 'terminal') this.disposeReason = 'terminal';
       return this.disposePromise;
     }
 
+    this.disposeReason = reason;
     this.state = 'disposing';
     this.disposePromise = (async () => {
       // Wait for all active embeddings to complete
       const maxWaitMs = 5000;
-      const startTime = Date.now();
-      while (this.activeEmbeddings > 0 && (Date.now() - startTime) < maxWaitMs) {
-        await new Promise(resolve => setTimeout(resolve, 50));
-      }
-
-      if (this.activeEmbeddings > 0) {
-        logger.warn(`[embedder] Disposing with ${this.activeEmbeddings} active embeddings (timed out)`);
-      }
+      const drainActiveEmbeddings = async (): Promise<void> => {
+        const startTime = Date.now();
+        while (this.activeEmbeddings > 0 && (Date.now() - startTime) < maxWaitMs) {
+          await new Promise(resolve => setTimeout(resolve, 50));
+        }
+        if (this.activeEmbeddings > 0) {
+          logger.warn(`[embedder] Disposing with ${this.activeEmbeddings} active embeddings (timed out)`);
+        }
+      };
+      await drainActiveEmbeddings();
 
       if (this.initializingPromise) {
         try {
@@ -556,6 +689,15 @@ export class Embedder {
         } catch (_e) {
           // Initialize failed, which is fine
         }
+        // A caller parked on that same init increments activeEmbeddings only
+        // AFTER init resolves — i.e. after the drain above already passed — and
+        // initialize()'s disposed-mid-init branch returns cleanly, so its embed
+        // proceeds into runInference. Yield one macrotask so those callers reach
+        // their increment, then drain AGAIN; otherwise the pipeline below is
+        // torn down under a just-started inference (an unclassified TypeError
+        // the writer queue drops as a generic ingest_error).
+        await new Promise(resolve => setTimeout(resolve, 0));
+        await drainActiveEmbeddings();
       }
 
       if (this.pipeline) {

package/src/knowledge/index.ts

@@ -115,7 +115,7 @@ export async function forceDeleteKnowledgeStore(config?: Config, workspace?: str
   const dbDir = getDbDir(config, workspace);
   if (fs.existsSync(dbDir)) {
     try {
-      fs.rmSync(dbDir, { recursive: true, force: true });
+      fs.rmSync(dbDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
       fs.mkdirSync(dbDir, { recursive: true });
       logger.info(`[knowledge] Knowledge store at ${dbDir} forcefully deleted and recreated.`);
     } catch (err) {

package/src/knowledge/store.ts

@@ -256,7 +256,12 @@ export class KnowledgeStore implements IKnowledgeStore {
             // getDimension() report the correct new width. (re-embed needs the warm
             // embedder regardless.)
             if (this.options.embedder.getDimension() === null) {
-              await this.options.embedder.embedMany([' ']);
+              // Reconnect-wrapped: after a service re-scope this warm-up can be
+              // the FIRST embed against a stale cached instance (a stepped-down
+              // leader fast-fails with ECONNREFUSED); without the wrapper that
+              // single fast-fail aborted the whole open, burning an init retry
+              // per attempt against the same dead instance.
+              await this.withEmbedderReconnect(embedder => embedder.embedMany([' ']));
             }
             try {
               await this.handleModelChange(storedModel || 'unknown', this.options.modelName, strategy);
@@ -721,12 +726,28 @@ export class KnowledgeStore implements IKnowledgeStore {
     }
   }
 
-  async deleteByUrlAndType(url: string, ingestionType: string): Promise<void> {
-    // Coordinate with close() — see deleteByUrl.
-    return this.trackOperation('deleteByUrlAndType', undefined, () => this.deleteByUrlAndTypeInner(url, ingestionType));
+  /**
+   * @param olderThan When set, only rows with `timestamp < olderThan` are removed.
+   *   This is what lets a refresh write the replacement chunks FIRST and prune the
+   *   superseded ones after, instead of deleting first and losing the entry
+   *   outright if the insert then fails. `timestamp` is used as the discriminator
+   *   because it is a real (indexed) column — `contentHash` lives inside the
+   *   JSON-encoded `metadata` blob and cannot be filtered on.
+   */
+  async deleteByUrlAndType(url: string, ingestionType: string, olderThan?: number): Promise<void> {
+    // Coordinate with close() — see deleteByUrl. The closing-skip is surfaced
+    // rather than silently resolving: a prune skipped because the store is
+    // closing (a post-run background ingest racing a Ctrl+C close) leaves
+    // duplicates exactly like a failed prune does, but resolved cleanly, so the
+    // writer-queue's failed-prune accounting never saw its most common cause.
+    if (this.isClosing) {
+      logger.warn(`[store] Prune of ${ingestionType} chunks for ${url} skipped — store is closing; superseded duplicates remain until the next refresh.`);
+      metrics.increment('knowledge_store_delete_total', 1, { operation: 'by_url_and_type', status: 'skipped_closing' });
+    }
+    return this.trackOperation('deleteByUrlAndType', undefined, () => this.deleteByUrlAndTypeInner(url, ingestionType, olderThan));
   }
 
-  private async deleteByUrlAndTypeInner(url: string, ingestionType: string): Promise<void> {
+  private async deleteByUrlAndTypeInner(url: string, ingestionType: string, olderThan?: number): Promise<void> {
     let retryCount = 0;
     const MAX_RETRIES = 5;
     const BASE_DELAY = 100;
@@ -738,11 +759,15 @@ export class KnowledgeStore implements IKnowledgeStore {
       try {
         const escapedUrl = url.replace(/'/g, "''");
         const escapedType = ingestionType.replace(/'/g, "''");
-        await table.delete(`url = '${escapedUrl}' AND ingestion_type = '${escapedType}' AND (${scopeFilter})`);
+        // Integer-coerced so the predicate can never carry anything but digits.
+        const ageClause = Number.isFinite(olderThan as number)
+          ? ` AND timestamp < ${Math.floor(olderThan as number)}`
+          : '';
+        await table.delete(`url = '${escapedUrl}' AND ingestion_type = '${escapedType}' AND (${scopeFilter})${ageClause}`);
         const duration = Date.now() - startTime;
         metrics.observe('knowledge_store_delete_duration_ms', duration, { operation: 'by_url_and_type' });
         metrics.increment('knowledge_store_delete_total', 1, { operation: 'by_url_and_type', status: 'success' });
-        logger.log(`[store] Deleted ${ingestionType} chunks for ${url} (scoped)`);
+        logger.log(`[store] Deleted ${ingestionType} chunks for ${url} (scoped)${ageClause ? ' older than the replacement' : ''}`);
         return;
       } catch (err) {
         if (isLanceCommitConflict(err)) {
@@ -842,10 +867,14 @@ export class KnowledgeStore implements IKnowledgeStore {
     const startTime = Date.now();
     const escapedUrl = url.replace(/'/g, "''");
 
+    // limit > 1 deliberately: after a failed prune two generations of this URL can
+    // coexist, and LanceDB's fragment-scan order typically returns the OLDEST row
+    // first — limit(1) would serve the superseded content (and feed its older
+    // timestamp to the serve-age gate) until the next successful ingest.
     const results = await table
       .query()
       .where(`url = '${escapedUrl}' AND ingestion_type = 'synthesis-description' AND (${scopeFilter})`)
-      .limit(1)
+      .limit(16)
       .toArray();
 
     const duration = Date.now() - startTime;
@@ -856,6 +885,8 @@ export class KnowledgeStore implements IKnowledgeStore {
       metrics.increment('knowledge_store_cache_hits_total', 1, { status: 'miss' });
       return null;
     }
+    // Newest generation wins, never row order.
+    results.sort((a: any, b: any) => Number(b.timestamp) - Number(a.timestamp));
 
     const r = results[0];
     // Read-time freshness. The stored timestamp (ms) is otherwise discarded here;
@@ -951,9 +982,19 @@ export class KnowledgeStore implements IKnowledgeStore {
         config: Index.fts(),
         replace: true,
       });
-      // Record the version AFTER indexing (createIndex commits its own transactions) so the
-      // next call compares against this post-rebuild baseline and only rebuilds on real writes.
-      this.lastFtsVersion = await (await this.getFreshTable()).version();
+      // Record the post-rebuild baseline from THIS handle, whose version() is
+      // checkout-stable: it advances with our own createIndex commits but not with
+      // commits from other handles/processes. Re-reading through a fresh handle
+      // here (the old code) read the LATEST version instead — addDocuments takes
+      // no lock, so rows committed DURING the rebuild were absorbed into the
+      // baseline as already-indexed and stayed out of the BM25 leg until an
+      // unrelated later commit. Each createIndex commits exactly one version
+      // (measured on lancedb 0.29, fresh and replace alike); anything beyond
+      // version+2 means a concurrent commit rebased beneath our index commits, so
+      // fall back to the conservative pre-rebuild baseline — the next cleanup
+      // sees version > baseline and rebuilds over the missed rows.
+      const postVersion = await table.version();
+      this.lastFtsVersion = postVersion <= version + 2 ? postVersion : version;
       logger.info('[store] FTS indexes rebuilt (text + content).');
       return true;
     } catch (err) {
@@ -1001,7 +1042,12 @@ export class KnowledgeStore implements IKnowledgeStore {
         // rebuilds + re-optimizes (the disk-bloat regression). A stale baseline (index not
         // current, or never built) is left untouched so the next rebuild still detects the
         // pending change.
-        const wasFtsIndexCurrent = !stale && this.lastFtsVersion !== null && this.lastFtsVersion === await table.version();
+        // The row count is sampled BEFORE the version read: since optimize never changes
+        // the row set, a same-handle count that differs after the call proves rows were
+        // committed DURING optimize's window (rebased beneath its compaction commits) —
+        // rows the FTS index does not cover, which the baseline advance below must skip.
+        const rowCountBefore = !stale && this.lastFtsVersion !== null ? await table.countRows() : null;
+        const wasFtsIndexCurrent = rowCountBefore !== null && this.lastFtsVersion === await table.version();
         // cleanupOlderThan defaults to now: prune every prunable old version. The
         // current version is always retained, and (with deleteUnverified=false)
         // physical files inside LanceDB's safety window are kept regardless.
@@ -1017,9 +1063,14 @@ export class KnowledgeStore implements IKnowledgeStore {
         const bytesRemoved = stats?.prune?.bytesRemoved ?? 0;
         logger.info(`[store] Optimize complete: removed ${removed} old version(s), reclaimed ${bytesRemoved} bytes (${duration}ms).`);
         // Keep the FTS skip-guard baseline in sync with optimize's layout-only version bump
-        // so the next rebuildFtsIndex correctly skips when no content changed.
-        if (wasFtsIndexCurrent) {
-          this.lastFtsVersion = await (await this.getFreshTable()).version();
+        // so the next rebuildFtsIndex correctly skips when no content changed. Stamp from
+        // THIS handle (checkout-stable: it advances with optimize's own commits but not
+        // with foreign ones — a fresh re-open here read the latest version and absorbed
+        // concurrent commits), and only when the row count is unchanged: a differing count
+        // means an addDocuments landed inside optimize's window and must not be recorded
+        // as already-indexed (the baseline stays put, so the next rebuild catches it).
+        if (wasFtsIndexCurrent && (await table.countRows()) === rowCountBefore) {
+          this.lastFtsVersion = await table.version();
         }
         return true;
       } catch (err) {
@@ -1081,20 +1132,28 @@ export class KnowledgeStore implements IKnowledgeStore {
       for (const name of migrationDirs) {
         const fullPath = path.join(this.options.dbDir, name);
         logger.info(`[store] Removing orphaned migration temp directory: ${name}`);
-        await fsPromises.rm(fullPath, { recursive: true, force: true });
+        await fsPromises.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
       }
 
       // `_backup_` dirs are intentional, user-recoverable snapshots from the
       // 'backup' migration strategy. RETAIN the most recent one (deleting it
       // would make 'backup' silently equivalent to 'drop'); prune only older,
-      // superseded backups so they don't accumulate. ISO-timestamp names sort
-      // chronologically, so the last entry is newest.
+      // superseded backups so they don't accumulate. Rank by the trailing
+      // `_backup_<timestamp>` suffix, NOT by whole-name sort: the prefix is the
+      // table name AT BACKUP TIME — legitimately `knowledge` OR
+      // `knowledge_migration_<ts>` (persisted re-embed rename failure) — and
+      // `knowledge_b…` < `knowledge_m…` lexicographically, so a whole-name sort
+      // retained an OLD migration-named backup while deleting a NEWER
+      // `knowledge_backup_*`.
       if (backupDirs.length > 1) {
-        backupDirs.sort();
-        for (const name of backupDirs.slice(0, -1)) {
+        const ranked = await Promise.all(
+          backupDirs.map(async name => ({ name, ts: await this.backupDirTimestamp(name) }))
+        );
+        ranked.sort((a, b) => a.ts - b.ts);
+        for (const { name } of ranked.slice(0, -1)) {
           const fullPath = path.join(this.options.dbDir, name);
           logger.info(`[store] Removing superseded backup directory: ${name}`);
-          await fsPromises.rm(fullPath, { recursive: true, force: true });
+          await fsPromises.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
         }
       }
     } catch (err) {
@@ -1102,6 +1161,27 @@ export class KnowledgeStore implements IKnowledgeStore {
     }
   }
 
+  /**
+   * Creation time of a backup dir, for retention ranking in
+   * pruneOrphanedMigrationDirs. Parses the trailing `_backup_<timestamp>` suffix
+   * (an ISO timestamp with `:`/`.` mapped to `-`, exactly as migrationBackup
+   * writes it); falls back to directory mtime for names that don't parse, and to
+   * 0 (ranked oldest, pruned first) if even stat fails.
+   */
+  private async backupDirTimestamp(name: string): Promise<number> {
+    const m = name.match(/_backup_(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z\.lance$/);
+    if (m) {
+      const parsed = Date.parse(`${m[1]}T${m[2]}:${m[3]}:${m[4]}.${m[5]}Z`);
+      if (!Number.isNaN(parsed)) return parsed;
+    }
+    try {
+      const stat = await fsPromises.stat(path.join(this.options.dbDir, name));
+      return stat.mtimeMs;
+    } catch {
+      return 0;
+    }
+  }
+
   async count(): Promise<number> {
     if (!this.db) return 0;
     // Track against close() — this read had no try/catch, so a teardown racing db.close()

package/src/knowledge/webgpu-viability.ts

@@ -150,10 +150,13 @@ function runProbe(model: string, cacheDir: string, timeoutMs: number): Promise<b
     }, timeoutMs);
     if (typeof timer.unref === 'function') timer.unref();
 
-    let stdout = '';
-    let stderr = '';
-    child.stdout?.on('data', (d) => { stdout += String(d); });
-    child.stderr?.on('data', (d) => { stderr += String(d); });
+    // Buffered and decoded once at 'close': per-chunk String(d) mangles a
+    // multi-byte character straddling a pipe-chunk boundary (the PROBE_OK
+    // sentinel is ASCII, but the logged stderr detail line is not guaranteed to be).
+    const stdoutChunks: Buffer[] = [];
+    const stderrChunks: Buffer[] = [];
+    child.stdout?.on('data', (d) => { stdoutChunks.push(Buffer.from(d)); });
+    child.stderr?.on('data', (d) => { stderrChunks.push(Buffer.from(d)); });
 
     child.on('error', (err) => {
       logger.warn('[embedder] WebGPU probe process error — defaulting to CPU:', err);
@@ -161,6 +164,8 @@ function runProbe(model: string, cacheDir: string, timeoutMs: number): Promise<b
     });
 
     child.on('close', (code, signal) => {
+      const stdout = Buffer.concat(stdoutChunks).toString('utf8');
+      const stderr = Buffer.concat(stderrChunks).toString('utf8');
       // Viability is the sentinel, not the exit code (teardown may abort post-success).
       if (stdout.includes('PROBE_OK')) {
         finish(true);