@vellumai/assistant 0.10.4-dev.202607010743.70cd9f0 → 0.10.4-dev.202607011113.fffc936

package/openapi.yaml

@@ -18822,7 +18822,8 @@ paths:
       description:
         "Enqueues the cursor-checkpointed backfill job that indexes existing messages into the Qdrant lexical
         (BM25-style) collection in batches. Resumable and idempotent — re-running continues from the last checkpoint.
-        Pass `force: true` to reset the cursor and re-index from the beginning. Not run at daemon startup."
+        Pass `force: true` to reset the cursor and re-index from the beginning. The same backfill is also auto-enqueued
+        once per instance on upgrade at assistant startup."
       tags:
         - memory
       requestBody:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.10.4-dev.202607010743.70cd9f0",
+  "version": "0.10.4-dev.202607011113.fffc936",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/context-search-conversations-source.test.ts

@@ -52,6 +52,11 @@ mock.module(
   }),
 );
 
+import {
+  deleteMemoryCheckpoint,
+  LEXICAL_BACKFILL_COMPLETE_KEY,
+  setMemoryCheckpoint,
+} from "../persistence/checkpoints.js";
 import { getDb } from "../persistence/db-connection.js";
 import { initializeDb } from "../persistence/db-init.js";
 import { rawRun } from "../persistence/raw-query.js";
@@ -402,12 +407,16 @@ describe("searchConversationSource with the qdrant backend", () => {
     getDb().run("DELETE FROM conversations");
     lexicalCalls = [];
     suppressIndexing = false;
+    // These tests exercise the populated-index (post-backfill) qdrant path, so
+    // mark the backfill complete. The completion gate is covered by its own test.
+    setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
     setOverridesForTesting({ "messages-search-backend": true });
   });
 
   afterEach(() => {
     setOverridesForTesting({});
     suppressIndexing = false;
+    deleteMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY);
     lexicalMockImpl = () => {
       throw new Error(
         "searchMessageIdsLexical mock not configured for this test",
@@ -442,6 +451,35 @@ describe("searchConversationSource with the qdrant backend", () => {
     ]);
   });
 
+  test("falls back to FTS until the backfill completion checkpoint is set, even with the qdrant flag on", async () => {
+    const match = await seedConversation({
+      title: "Pre-backfill notes",
+      content: "The prebackfilltoken decision is recorded here.",
+    });
+
+    // On an upgraded instance the historical messages are still being indexed
+    // by the background backfill, so the lexical collection is only partially
+    // populated. Until the completion checkpoint is set, the source must use
+    // the FTS path and never query Qdrant.
+    deleteMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY);
+    lexicalMockImpl = async () => {
+      throw new Error(
+        "searchMessageIdsLexical must not run before the backfill completes",
+      );
+    };
+
+    const result = await searchConversationSource(
+      "prebackfilltoken",
+      makeContext(),
+      5,
+    );
+
+    expect(lexicalCalls).toHaveLength(0);
+    expect(result.evidence.map((item) => item.locator)).toEqual([
+      `${match.conversation.id}#${match.message.id}`,
+    ]);
+  });
+
   test("routes short/punctuation-only queries to the exact LIKE path, not Qdrant", async () => {
     const match = await seedConversation({
       title: "C++ notes",

package/src/__tests__/plugin-import-boundary-guard.test.ts

@@ -82,6 +82,7 @@ const BASELINE: Record<string, readonly string[]> = {
     "../../../../../config/types.js",
     "../../../../../messaging/providers/slack/message-metadata.js",
     "../../../../../persistence/auto-analysis-constants.js",
+    "../../../../../persistence/checkpoints.js",
     "../../../../../persistence/conversation-queries.js",
     "../../../../../persistence/conversation-search-lexical.js",
     "../../../../../persistence/db-connection.js",

package/src/daemon/lifecycle.ts

@@ -31,6 +31,7 @@ import {
 } from "../persistence/conversation-crud.js";
 import { getDb } from "../persistence/db-connection.js";
 import { initializeDb } from "../persistence/db-init.js";
+import { startEmbeddingRuntimeManager } from "../persistence/embeddings/embedding-runtime-manager.js";
 import { startConsentRefresh } from "../platform/consent-cache.js";
 import { syncWorkspaceIdentityToPlatform } from "../platform/sync-identity.js";
 import { runMemoryStartup } from "../plugins/defaults/memory/startup.js";
@@ -879,30 +880,7 @@ export async function runDaemon(): Promise<void> {
     log.warn({ err }, "Assistant symlink installation failed — continuing");
   }
 
-  // Download embedding runtime in background (non-blocking).
-  // If download fails, local embeddings gracefully fall back to cloud backends.
-  void (async () => {
-    try {
-      const { EmbeddingRuntimeManager } =
-        await import("../persistence/embeddings/embedding-runtime-manager.js");
-      const runtimeManager = new EmbeddingRuntimeManager();
-      if (!runtimeManager.isReady()) {
-        log.info("Downloading embedding runtime in background...");
-        await runtimeManager.ensureInstalled();
-        // Reset the sticky local-backend failure flag so auto mode retries
-        // local embeddings without evicting a worker that may already be live.
-        const { resetLocalEmbeddingFailureState } =
-          await import("../persistence/embeddings/embedding-backend.js");
-        resetLocalEmbeddingFailureState();
-        log.info("Embedding runtime download complete");
-      }
-    } catch (err) {
-      log.warn(
-        { err },
-        "Embedding runtime download failed — local embeddings will use cloud fallback",
-      );
-    }
-  })();
+  startEmbeddingRuntimeManager();
 
   startWorkspaceHeartbeatService();
 

package/src/persistence/__tests__/conversation-queries-search.test.ts

@@ -53,6 +53,11 @@ mock.module("../jobs-store.js", () => ({
 }));
 
 import { setOverridesForTesting } from "../../__tests__/feature-flag-test-helpers.js";
+import {
+  deleteMemoryCheckpoint,
+  LEXICAL_BACKFILL_COMPLETE_KEY,
+  setMemoryCheckpoint,
+} from "../checkpoints.js";
 import { createConversation } from "../conversation-crud.js";
 import { searchConversations } from "../conversation-queries.js";
 import { getDb } from "../db-connection.js";
@@ -61,6 +66,17 @@ import { rawRun } from "../raw-query.js";
 
 await initializeDb();
 
+/**
+ * Mark the messages lexical-index backfill complete. The qdrant read path is
+ * gated on this: an upgraded instance whose backfill has not finished stays on
+ * fts5 so it never reads from a partially-populated `messages_lexical`. The
+ * qdrant-path tests below index candidates as if the backfill has drained, so
+ * they set this marker; the gate itself is covered by its own test.
+ */
+function markBackfillComplete(): void {
+  setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
+}
+
 function resetTables(): void {
   const db = getDb();
   db.run(`DELETE FROM messages`);
@@ -118,6 +134,8 @@ describe("searchConversations · qdrant backend", () => {
   beforeEach(() => {
     resetTables();
     memoryEnabled = true;
+    // These tests exercise the populated-index (post-backfill) qdrant path.
+    markBackfillComplete();
     searchMessageIdsLexicalMock.mockClear();
     searchMessageIdsLexicalMock.mockImplementation(async () => []);
     setOverridesForTesting({ "messages-search-backend": "qdrant" });
@@ -313,6 +331,27 @@ describe("searchConversations · qdrant backend", () => {
     ]);
   });
 
+  test("uses the fts5 path (not qdrant) until the backfill completion checkpoint is set", async () => {
+    // On an upgraded instance the historical messages are indexed by a
+    // background backfill. Until it drains, the lexical collection is only
+    // partially populated, so a qdrant read would silently miss older content.
+    // With the flag on but the completion checkpoint UNSET, the read must stay
+    // on the always-populated fts5 path and never query the lexical index.
+    deleteMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY);
+    const conv = createConversation("Notes");
+    insertMessage("m-1", conv.id, "the flux capacitor needs recalibration");
+    lexicalReturns(["m-1"]);
+
+    const results = await searchConversations("flux capacitor");
+
+    expect(searchMessageIdsLexicalMock).not.toHaveBeenCalled();
+    // The fts5 content match still finds the conversation and its message.
+    expect(results.map((r) => r.conversationId)).toEqual([conv.id]);
+    expect(results[0]!.matchingMessages.map((m) => m.messageId)).toEqual([
+      "m-1",
+    ]);
+  });
+
   test("non-tokenizable queries use the LIKE fallback without hitting the index", async () => {
     // Single-char / non-ASCII queries produce no tokens, so neither FTS nor the
     // sparse encoder yields terms — both backends use the LIKE content scan.
@@ -338,6 +377,9 @@ describe("searchConversations · qdrant backend", () => {
 describe("searchConversations · fts5 backend (default) ignores the lexical index", () => {
   beforeEach(() => {
     resetTables();
+    // Backfill completion is irrelevant on the default backend, but clear it so
+    // this suite does not depend on the qdrant suite's marker leaking across.
+    deleteMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY);
     searchMessageIdsLexicalMock.mockClear();
     lexicalReturns(["should-not-be-used"]);
     // Default backend: flag unset ⇒ fts5.

package/src/persistence/checkpoints.ts

@@ -63,3 +63,48 @@ export function resetMessageCursorCheckpoint(
     messageId: "",
   });
 }
+
+/**
+ * Completion sentinel for the messages lexical-index backfill. Set once the
+ * backfill has drained the entire `messages` table into the Qdrant lexical
+ * index (`messages_lexical`); cleared on a `force` re-run.
+ *
+ * Lives here — in `persistence/` — so it is the single source of truth for the
+ * backfill handler (plugin layer, which writes it) AND the message-search read
+ * sites (persistence + plugin layers, which gate the read backend on it).
+ * Keeping the key and its reader here lets `conversation-queries.ts` (also
+ * `persistence/`) read it without a persistence -> plugin import.
+ */
+export const LEXICAL_BACKFILL_COMPLETE_KEY =
+  "lexical:messages:backfill_complete";
+
+/**
+ * True once the messages lexical-index backfill has fully drained on this
+ * instance.
+ *
+ * Gates two things: the one-time startup auto-enqueue (skip when already
+ * complete) and — critically — the message-search read backend. An upgraded
+ * instance whose backfill has not finished must keep reading from SQLite FTS,
+ * because a `qdrant`-backed read against the still-filling `messages_lexical`
+ * collection returns an empty result (not a throw), so the Qdrant-error degrade
+ * path never fires and content search would silently return nothing. Switch
+ * reads to Qdrant only once this marker confirms the index is fully populated.
+ */
+export function isLexicalBackfillComplete(): boolean {
+  return getMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY) === "1";
+}
+
+/**
+ * Clear the lexical-backfill completion sentinel so {@link
+ * isLexicalBackfillComplete} reads false until a run re-marks it.
+ *
+ * Called when a `force` re-index is requested — both at enqueue time (so reads
+ * fall back to SQLite FTS the instant the rebuild is queued, rather than serving
+ * from a stale/emptying `messages_lexical` collection in the window before the
+ * worker claims the job) and again inside the backfill handler (idempotent).
+ * Co-located with the key and its reader here so callers clear it without
+ * re-declaring the key string.
+ */
+export function clearLexicalBackfillComplete(): void {
+  deleteMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY);
+}

package/src/persistence/conversation-queries.ts

@@ -9,6 +9,7 @@ import {
   wrapUntrustedContent,
 } from "../security/untrusted-content.js";
 import { getLogger } from "../util/logger.js";
+import { isLexicalBackfillComplete } from "./checkpoints.js";
 import type { ConversationRow } from "./conversation-crud.js";
 import { parseConversation } from "./conversation-crud.js";
 import { ensureDisplayOrderMigration } from "./conversation-display-order-migration.js";
@@ -553,11 +554,36 @@ function likeContentMatchMessages(
   );
 }
 
+/**
+ * Resolve the effective message-search backend for the persistence read site.
+ *
+ * Starts from the `messages-search-backend` feature flag, then downgrades to
+ * `fts5` in two cases where the Qdrant `messages_lexical` collection is not a
+ * safe source (each would return an empty result — not a throw — so the
+ * Qdrant-error degrade never fires):
+ *   1. Memory is disabled — the per-message write path is suppressed, so the
+ *      collection is never forward-filled. `!isMemoryEnabled()` is the
+ *      layer-clean check available from `persistence/`.
+ *   2. The one-time upgrade backfill has not finished — the collection is only
+ *      partially populated, so older content is missing until it drains.
+ *
+ * The recall read site applies the same completion gate (via the shared
+ * {@link isLexicalBackfillComplete}) on top of its own, stricter suppression
+ * check (`isMemoryIndexingSuppressed`, which also covers a disabled plugin).
+ */
+function resolveMessageSearchBackend(): "fts5" | "qdrant" {
+  if (!isMemoryEnabled()) return "fts5";
+  const flagBackend = getMessagesSearchBackend(getConfig());
+  if (flagBackend === "qdrant" && !isLexicalBackfillComplete()) return "fts5";
+  return flagBackend;
+}
+
 /**
  * Full-text search across message content.
  *
  * The lexical backend is selected by the `messages-search-backend` feature
- * flag (see {@link getMessagesSearchBackend}):
+ * flag (see {@link getMessagesSearchBackend}) and gated by
+ * {@link resolveMessageSearchBackend}:
  *   - `fts5` (default): the `messages_fts` virtual table for tokenized matching.
  *   - `qdrant`: the sparse `messages_lexical` Qdrant index (BM25-style).
  * Both apply the same visibility/archived SQL filtering, merge with a `LIKE`
@@ -600,9 +626,14 @@ export async function searchConversations(
   // `isMemoryIndexingSuppressed` predicate) can't be reached from here without a
   // persistence -> plugin import, and is covered at flip time by requiring the
   // backfill/population check before enabling the flag.
-  const backend = !isMemoryEnabled()
-    ? "fts5"
-    : getMessagesSearchBackend(getConfig());
+  //
+  // The backfill completion gate is the second half of that population guard,
+  // for the common (memory-enabled) case: on an upgraded instance the historical
+  // messages are indexed by a background backfill, so until it has fully drained
+  // the `messages_lexical` collection is only partially populated and a `qdrant`
+  // read would silently miss older content. Stay on `fts5` until
+  // `isLexicalBackfillComplete()` confirms the index is whole, then switch.
+  const backend = resolveMessageSearchBackend();
 
   // LIKE pattern for title matching (message-content indexes don't cover titles).
   const titlePattern = likeContainsPattern(query);

package/src/persistence/embeddings/embedding-runtime-manager.ts

@@ -571,3 +571,34 @@ export class EmbeddingRuntimeManager {
     }
   }
 }
+
+/**
+ * Download the local embedding runtime in the background (non-blocking).
+ *
+ * Pre-warms the runtime (bun binary + ONNX worker scripts) so the first local
+ * embed doesn't pay the download cost inline. A no-op when the runtime is
+ * already installed. Failures are swallowed with a warning: local embeddings
+ * fall back to cloud backends, so a failed download must never block or crash
+ * daemon startup. Returns immediately; the download proceeds on its own.
+ */
+export function startEmbeddingRuntimeManager(): void {
+  void (async () => {
+    try {
+      const runtimeManager = new EmbeddingRuntimeManager();
+      if (runtimeManager.isReady()) return;
+      log.info("Downloading embedding runtime in background...");
+      await runtimeManager.ensureInstalled();
+      // Reset the sticky local-backend failure flag so auto mode retries
+      // local embeddings without evicting a worker that may already be live.
+      const { resetLocalEmbeddingFailureState } =
+        await import("./embedding-backend.js");
+      resetLocalEmbeddingFailureState();
+      log.info("Embedding runtime download complete");
+    } catch (err) {
+      log.warn(
+        { err },
+        "Embedding runtime download failed — local embeddings will use cloud fallback",
+      );
+    }
+  })();
+}

package/src/plugins/defaults/memory/context-search/sources/conversations.ts

@@ -1,6 +1,7 @@
 import { getMessagesSearchBackend } from "../../../../../config/assistant-feature-flags.js";
 import { readSlackMetadata } from "../../../../../messaging/providers/slack/message-metadata.js";
 import { AUTO_ANALYSIS_SOURCE } from "../../../../../persistence/auto-analysis-constants.js";
+import { isLexicalBackfillComplete } from "../../../../../persistence/checkpoints.js";
 import {
   buildFtsMatchQuery,
   buildRecallEvidenceExcerpt,
@@ -128,9 +129,17 @@ export async function searchConversationSource(
   // (memory disabled or the memory plugin disabled) the collection is never
   // populated, so a qdrant-backed read would silently return nothing while FTS
   // still works — force the FTS path regardless of the flag in that case.
-  const backend = isMemoryIndexingSuppressed()
-    ? "fts5"
-    : getMessagesSearchBackend(context.config);
+  //
+  // Second gate: until the one-time upgrade backfill has fully drained, the
+  // collection holds only messages written since the write path went live —
+  // older history is missing. Reading from Qdrant then silently misses that
+  // history (an empty candidate set, not a throw), so stay on FTS until
+  // `isLexicalBackfillComplete()` confirms the index is whole. Shared with the
+  // persistence read site via the same checkpoint helper.
+  const backend =
+    isMemoryIndexingSuppressed() || !isLexicalBackfillComplete()
+      ? "fts5"
+      : getMessagesSearchBackend(context.config);
 
   // Tokenize once. Short/punctuation-only queries (`C++`, CJK) produce no
   // usable ≥2-char match shape, and both backends must route those straight to

package/src/plugins/defaults/memory/job-handlers/__tests__/backfill-lexical-index.test.ts

@@ -69,12 +69,24 @@ mock.module(
 // `generateSparseEmbedding` is a pure local TF-IDF encoder (no provider call),
 // so it runs unmocked — mocking `embedding-backend.js` wholesale would starve
 // its other named exports that the db-init import graph pulls in.
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
 import { resetDbForTesting } from "../../../../../__tests__/db-test-helpers.js";
 import { DEFAULT_CONFIG } from "../../../../../config/defaults.js";
+import {
+  invalidateConfigCache,
+  loadRawConfig,
+  saveRawConfig,
+} from "../../../../../config/loader.js";
 import type { AssistantConfig } from "../../../../../config/types.js";
 import {
+  deleteMemoryCheckpoint,
+  isLexicalBackfillComplete,
+  LEXICAL_BACKFILL_COMPLETE_KEY,
   readMessageCursorCheckpoint,
   resetMessageCursorCheckpoint,
+  setMemoryCheckpoint,
   writeMessageCursorCheckpoint,
 } from "../../../../../persistence/checkpoints.js";
 import {
@@ -89,7 +101,17 @@ import {
   memoryJobs,
   messages,
 } from "../../../../../persistence/schema/index.js";
-import { backfillLexicalIndexJob } from "../backfill-lexical-index.js";
+import type {
+  RouteDefinition,
+  RouteHandlerArgs,
+} from "../../../../../runtime/routes/types.js";
+import { getWorkspacePluginsDir } from "../../../../../util/platform.js";
+import memoryPkg from "../../package.json" with { type: "json" };
+import { ROUTES as MESSAGES_LEXICAL_ROUTES } from "../../routes/messages-lexical-routes.js";
+import {
+  backfillLexicalIndexJob,
+  maybeEnqueueLexicalBackfillOnUpgrade,
+} from "../backfill-lexical-index.js";
 
 const TEST_CONFIG: AssistantConfig = DEFAULT_CONFIG;
 
@@ -147,6 +169,55 @@ function pendingBackfillJobCount(): number {
   return rows.length;
 }
 
+/**
+ * Invoke the `messages_lexical_backfill` route handler directly through its
+ * shared `ROUTES` entry (transport-agnostic), the same way the HTTP/IPC adapters
+ * call it. Returns the handler result (`{ jobId }`).
+ */
+async function callBackfillRoute(
+  body: Record<string, unknown>,
+): Promise<{ jobId: string }> {
+  const route = MESSAGES_LEXICAL_ROUTES.find(
+    (r: RouteDefinition) => r.operationId === "messages_lexical_backfill",
+  );
+  if (!route) throw new Error("messages_lexical_backfill route not found");
+  const args: RouteHandlerArgs = { body };
+  return (await route.handler(args)) as { jobId: string };
+}
+
+/**
+ * Toggle `memory.enabled` in the real workspace config so the startup hook's
+ * `isMemoryEnabled()` (which reads `getConfig()`) sees the change. Driving the
+ * real config instead of mocking `jobs-store` avoids a process-global
+ * `mock.module` leaking `isMemoryEnabled` into sibling test files.
+ */
+function setMemoryEnabled(enabled: boolean): void {
+  const raw = loadRawConfig();
+  const memory =
+    raw.memory && typeof raw.memory === "object"
+      ? (raw.memory as Record<string, unknown>)
+      : {};
+  saveRawConfig({ ...raw, memory: { ...memory, enabled } });
+  invalidateConfigCache();
+}
+
+/**
+ * Toggle the memory plugin's `.disabled` sentinel in the real workspace plugins
+ * dir so `isMemoryIndexingSuppressed()` (via `isPluginDisabled(memoryPkg.name)`)
+ * sees the change. Driving the real sentinel — not a process-global mock —
+ * exercises the same suppression path the write/recall gates use and avoids
+ * leaking a mock into sibling test files.
+ */
+function setMemoryPluginDisabled(disabled: boolean): void {
+  const dir = join(getWorkspacePluginsDir(), memoryPkg.name);
+  if (disabled) {
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, ".disabled"), "");
+  } else {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
 describe("backfillLexicalIndexJob", () => {
   // initializeDb runs the full migration chain; under parallel CI load it can
   // exceed bun's default 5s hook timeout, so allow more.
@@ -157,6 +228,8 @@ describe("backfillLexicalIndexJob", () => {
   beforeEach(async () => {
     upsertBatchCalls.length = 0;
     resetLexicalSingleton(true);
+    setMemoryEnabled(true);
+    setMemoryPluginDisabled(false);
     resetDbForTesting();
     await initializeDb();
     // The template-restore path leaves stale WAL sidecars, so rows can bleed
@@ -166,6 +239,7 @@ describe("backfillLexicalIndexJob", () => {
     getDb().delete(conversations).run();
     getMemoryDb()!.delete(memoryJobs).run();
     resetMessageCursorCheckpoint(CHECKPOINT_KEY, CHECKPOINT_ID_KEY);
+    deleteMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY);
   }, 30_000);
 
   test("indexes a batch of messages and advances the checkpoint", async () => {
@@ -202,8 +276,9 @@ describe("backfillLexicalIndexJob", () => {
     );
     expect(cursor).toEqual({ createdAt: 2_000, messageId: "msg-b" });
 
-    // A short batch does not re-enqueue.
+    // A short batch does not re-enqueue, and marks the backfill complete.
     expect(pendingBackfillJobCount()).toBe(0);
+    expect(isLexicalBackfillComplete()).toBe(true);
   });
 
   test("resumes from the persisted cursor, skipping already-indexed rows", async () => {
@@ -251,12 +326,18 @@ describe("backfillLexicalIndexJob", () => {
     expect(upsertBatchCalls[0]).toHaveLength(200);
     // Exactly one follow-up job enqueued to process the remaining rows.
     expect(pendingBackfillJobCount()).toBe(1);
+    // A full batch means more rows may remain, so the backfill is NOT yet
+    // complete — the completion marker must stay unset until a short batch.
+    expect(isLexicalBackfillComplete()).toBe(false);
   });
 
-  test("does nothing (no upsert, no re-enqueue) when there are no messages", async () => {
+  test("does nothing (no upsert, no re-enqueue) when there are no messages, and marks complete", async () => {
     await backfillLexicalIndexJob(makeJob({}), TEST_CONFIG);
     expect(upsertBatchCalls).toHaveLength(0);
     expect(pendingBackfillJobCount()).toBe(0);
+    // An empty batch is a drained backfill (an already-empty or fully-indexed
+    // table) — the completion marker is written.
+    expect(isLexicalBackfillComplete()).toBe(true);
   });
 
   test("force resets the cursor and re-indexes from the beginning", async () => {
@@ -268,11 +349,13 @@ describe("backfillLexicalIndexJob", () => {
       createdAt: 5_000,
     });
 
-    // A stale cursor past every row would normally skip everything.
+    // A stale cursor past every row would normally skip everything, and a prior
+    // run had already marked the backfill complete.
     writeMessageCursorCheckpoint(CHECKPOINT_KEY, CHECKPOINT_ID_KEY, {
       createdAt: 9_999,
       messageId: "msg-zzz",
     });
+    setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
 
     await backfillLexicalIndexJob(makeJob({ force: true }), TEST_CONFIG);
 
@@ -286,6 +369,32 @@ describe("backfillLexicalIndexJob", () => {
       CHECKPOINT_ID_KEY,
     );
     expect(cursor).toEqual({ createdAt: 5_000, messageId: "msg-x" });
+
+    // force cleared the completion marker; the short re-index batch then drained
+    // and re-marked it complete.
+    expect(isLexicalBackfillComplete()).toBe(true);
+  });
+
+  test("force clears the completion marker even when the re-run does not drain in one batch", async () => {
+    insertConversation("conv-force-full");
+    // A full batch on the forced run: the marker must be CLEARED (re-armed) and
+    // stay unset because more rows may remain past this batch.
+    for (let i = 0; i < 200; i++) {
+      insertMessage({
+        id: `fm-${String(i).padStart(4, "0")}`,
+        conversationId: "conv-force-full",
+        content: `forced message ${i}`,
+        createdAt: 1_000 + i,
+      });
+    }
+    setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
+
+    await backfillLexicalIndexJob(makeJob({ force: true }), TEST_CONFIG);
+
+    expect(upsertBatchCalls[0]).toHaveLength(200);
+    expect(pendingBackfillJobCount()).toBe(1);
+    // Marker was cleared by force and not re-set (batch was full → more remain).
+    expect(isLexicalBackfillComplete()).toBe(false);
   });
 
   test("lazily initializes the index when not yet initialized (worker process)", async () => {
@@ -307,4 +416,108 @@ describe("backfillLexicalIndexJob", () => {
     expect(upsertBatchCalls).toHaveLength(1);
     expect(upsertBatchCalls[0].map((p) => p.messageId)).toEqual(["msg-w"]);
   });
+
+  // The one-time startup auto-enqueue. Inherits the outer beforeEach, which
+  // resets the DB, clears the memory_jobs table and completion marker, and sets
+  // memoryEnabled = true — so each test starts from a fresh, memory-enabled,
+  // never-backfilled instance.
+  describe("maybeEnqueueLexicalBackfillOnUpgrade", () => {
+    test("enqueues exactly one backfill job when memory is enabled and the checkpoint is unset", () => {
+      maybeEnqueueLexicalBackfillOnUpgrade();
+
+      const rows = getMemoryDb()!
+        .select({ type: memoryJobs.type })
+        .from(memoryJobs)
+        .all();
+      expect(rows).toHaveLength(1);
+      expect(rows[0].type).toBe("backfill_lexical_index");
+    });
+
+    test("does NOT enqueue when the completion checkpoint is already set", () => {
+      setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
+
+      maybeEnqueueLexicalBackfillOnUpgrade();
+
+      expect(pendingBackfillJobCount()).toBe(0);
+    });
+
+    test("does NOT enqueue when memory is disabled", () => {
+      setMemoryEnabled(false);
+
+      maybeEnqueueLexicalBackfillOnUpgrade();
+
+      expect(pendingBackfillJobCount()).toBe(0);
+    });
+
+    // Indexing is suppressed by the memory plugin's `.disabled` sentinel even
+    // while `memory.enabled` is true. The hook gates on the same
+    // `isMemoryIndexingSuppressed` signal as the write/recall paths, so it must
+    // skip: enqueuing (and later setting the completion marker) would leave a
+    // stale index that a lexical-backed read could serve while per-message writes
+    // stay suppressed.
+    test("does NOT enqueue when the memory plugin is disabled but memory.enabled is true", () => {
+      setMemoryEnabled(true);
+      setMemoryPluginDisabled(true);
+
+      maybeEnqueueLexicalBackfillOnUpgrade();
+
+      expect(pendingBackfillJobCount()).toBe(0);
+      // The completion marker must stay unset so reads stay on FTS.
+      expect(isLexicalBackfillComplete()).toBe(false);
+    });
+
+    test("does NOT enqueue a duplicate when a backfill job is already pending", () => {
+      // First call enqueues the one-time job.
+      maybeEnqueueLexicalBackfillOnUpgrade();
+      expect(pendingBackfillJobCount()).toBe(1);
+
+      // A second call (e.g. a restart mid-backfill) must not pile on a duplicate
+      // while the prior job is still pending.
+      maybeEnqueueLexicalBackfillOnUpgrade();
+      expect(pendingBackfillJobCount()).toBe(1);
+    });
+  });
+
+  // The manual backfill route (`handleBackfillLexicalIndex`). Inherits the outer
+  // beforeEach, which resets the DB, clears the memory_jobs table and completion
+  // marker, and enables memory — so each test starts from a fresh instance.
+  describe("handleBackfillLexicalIndex route", () => {
+    test("force: true clears the completion marker synchronously at enqueue, before the job runs", async () => {
+      // Pretend a prior backfill already fully drained on this instance, so
+      // reads would be serving from the lexical index.
+      setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
+      expect(isLexicalBackfillComplete()).toBe(true);
+
+      const { jobId } = await callBackfillRoute({ force: true });
+      expect(jobId).toBeTruthy();
+
+      // The marker is cleared the instant the forced rebuild is enqueued —
+      // before the worker claims the job — so reads immediately fall back to
+      // FTS instead of serving from the stale/emptying lexical index.
+      expect(isLexicalBackfillComplete()).toBe(false);
+      // The job was enqueued with the force payload for the worker to process.
+      expect(pendingBackfillJobCount()).toBe(1);
+    });
+
+    test("non-force enqueue does NOT clear the completion marker", async () => {
+      // A completed instance stays on the lexical read path for a resume-style
+      // (non-force) enqueue — only a forced rebuild re-arms the FTS fallback.
+      setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
+      expect(isLexicalBackfillComplete()).toBe(true);
+
+      const { jobId } = await callBackfillRoute({});
+      expect(jobId).toBeTruthy();
+
+      expect(isLexicalBackfillComplete()).toBe(true);
+      expect(pendingBackfillJobCount()).toBe(1);
+    });
+
+    test("force: false is treated as non-force and leaves the marker set", async () => {
+      setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
+
+      await callBackfillRoute({ force: false });
+
+      expect(isLexicalBackfillComplete()).toBe(true);
+    });
+  });
 });

package/src/plugins/defaults/memory/job-handlers/backfill-lexical-index.ts

@@ -2,8 +2,12 @@ import { and, asc, eq, gt, or } from "drizzle-orm";
 
 import type { AssistantConfig } from "../../../../config/types.js";
 import {
+  clearLexicalBackfillComplete,
+  isLexicalBackfillComplete,
+  LEXICAL_BACKFILL_COMPLETE_KEY,
   readMessageCursorCheckpoint,
   resetMessageCursorCheckpoint,
+  setMemoryCheckpoint,
   writeMessageCursorCheckpoint,
 } from "../../../../persistence/checkpoints.js";
 import { getDb } from "../../../../persistence/db-connection.js";
@@ -11,10 +15,17 @@ import { generateSparseEmbedding } from "../../../../persistence/embeddings/embe
 import { withQdrantBreaker } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
 import {
   enqueueMemoryJob,
+  hasActiveJobOfType,
   type MemoryJob,
 } from "../../../../persistence/jobs-store.js";
 import { messages } from "../../../../persistence/schema/index.js";
-import { resolveLexicalIndex } from "./index-message-lexical.js";
+import { getLogger } from "../../../../util/logger.js";
+import {
+  isMemoryIndexingSuppressed,
+  resolveLexicalIndex,
+} from "./index-message-lexical.js";
+
+const log = getLogger("lexical-backfill");
 
 /**
  * Cursor checkpoint keys for the lexical-index backfill. Kept distinct from the
@@ -25,6 +36,66 @@ import { resolveLexicalIndex } from "./index-message-lexical.js";
 const LEXICAL_BACKFILL_CHECKPOINT_KEY = "lexical:messages:last_created_at";
 const LEXICAL_BACKFILL_CHECKPOINT_ID_KEY = "lexical:messages:last_id";
 
+/**
+ * One-time, self-healing auto-enqueue of the messages lexical-index backfill,
+ * invoked once from `runMemoryStartup` (startup.ts) on daemon boot. Ensures each
+ * instance populates its Qdrant lexical index (`messages_lexical`) exactly once
+ * on upgrade — in the background — so a later read-path flip to the lexical
+ * backend does not open onto an empty index.
+ *
+ * Deliberate, narrow exception to the "not run at daemon startup" note carried by
+ * the manual backfill route (`messages-lexical-routes.ts`, added in the FTS→Qdrant
+ * PR 5): this call only *enqueues* a memory job — it does no indexing, no LLM
+ * work, and no other heavy lifting on the boot thread. The backfill itself runs
+ * off the event loop via the existing memory job worker, in small
+ * cursor-checkpointed batches. It is a one-time, checkpoint-guarded, no-LLM data
+ * backfill (migration-like), so it is consistent with the daemon-startup
+ * philosophy of never blocking or doing expensive work at boot.
+ *
+ * Guards (all must pass to enqueue):
+ * - Memory indexing must not be suppressed — memory enabled AND the memory
+ *   plugin not disabled — using the same {@link isMemoryIndexingSuppressed}
+ *   signal as the write/recall paths. When `memory.enabled === false` the memory
+ *   job worker drains nothing (`runMemoryJobsOnce` returns early), so an enqueued
+ *   backfill would sit pending forever and only accumulate dead rows. When the
+ *   memory plugin is disabled via its `.disabled` sentinel, per-message index
+ *   writes are suppressed, so completing a backfill would leave a stale index
+ *   that a lexical-backed read could serve; gating on the same signal keeps the
+ *   completion marker unset while writes are suppressed, so every read path stays
+ *   on FTS in that state (the marker unifies them).
+ * - The completion sentinel must be unset. Once the backfill has fully drained
+ *   on this instance there is nothing to do; the marker makes this idempotent
+ *   across restarts.
+ * - No backfill job may already be pending or running. Without this, every
+ *   restart during a long backfill would pile up a duplicate job. (The batches
+ *   are idempotent, but redundant jobs waste worker cycles.)
+ *
+ * An instance that was already manually backfilled via the route but predates
+ * the completion sentinel re-enqueues once here and re-runs the idempotent
+ * backfill (no duplicate points; ~minutes) — an acceptable one-time cost.
+ *
+ * Best-effort and lightweight: any failure (e.g. the memory database is briefly
+ * unavailable at boot) is swallowed and logged so a memory hiccup never blocks
+ * daemon startup, matching the surrounding startup steps.
+ */
+export function maybeEnqueueLexicalBackfillOnUpgrade(): void {
+  try {
+    if (isMemoryIndexingSuppressed()) return;
+    if (isLexicalBackfillComplete()) return;
+    if (hasActiveJobOfType("backfill_lexical_index")) return;
+    const jobId = enqueueMemoryJob("backfill_lexical_index", {});
+    log.info(
+      { jobId },
+      "Enqueued one-time messages lexical-index backfill on startup",
+    );
+  } catch (err) {
+    log.warn(
+      { err },
+      "Failed to enqueue startup lexical-index backfill (non-fatal)",
+    );
+  }
+}
+
 /** Messages indexed per job invocation before the job re-enqueues itself. */
 const BATCH_SIZE = 200;
 
@@ -51,6 +122,12 @@ export async function backfillLexicalIndexJob(
       LEXICAL_BACKFILL_CHECKPOINT_KEY,
       LEXICAL_BACKFILL_CHECKPOINT_ID_KEY,
     );
+    // Clearing the completion sentinel lets a forced re-run drain and re-mark
+    // as complete, and re-arms the one-time startup auto-enqueue for this run.
+    // The route also clears this at enqueue time (so reads fall back to FTS
+    // before the worker claims the job); clearing again here is idempotent and
+    // keeps the forced-run semantics self-contained in the handler.
+    clearLexicalBackfillComplete();
   }
 
   const cursor = readMessageCursorCheckpoint(
@@ -104,5 +181,12 @@ export async function backfillLexicalIndexJob(
   // happened this invocation, and re-resetting would loop over the same head.
   if (batch.length === BATCH_SIZE) {
     enqueueMemoryJob("backfill_lexical_index", {});
+  } else {
+    // A short batch (including an empty one) means the cursor has reached the
+    // end of the `messages` table — the backfill has fully drained. Record the
+    // completion sentinel so the one-time startup auto-enqueue never schedules
+    // it again on this instance. Idempotent: re-running after completion draws
+    // an empty batch and simply re-sets the same marker.
+    setMemoryCheckpoint(LEXICAL_BACKFILL_COMPLETE_KEY, "1");
   }
 }

package/src/plugins/defaults/memory/routes/messages-lexical-routes.ts

@@ -5,12 +5,15 @@
  * Deliberately NOT gated on `memory.v2.enabled`: the lexical index is a
  * BM25-style full-text replacement for message search that is independent of
  * the v2 concept-page machinery. The backfill runs off the event loop as a
- * cursor-checkpointed memory job and is never triggered at daemon startup — an
- * operator (or the client) enqueues it on demand.
+ * cursor-checkpointed memory job. This route enqueues it on demand (operator or
+ * client); the one-time, checkpoint-guarded startup auto-enqueue in
+ * `runMemoryStartup` (see `maybeEnqueueLexicalBackfillOnUpgrade`) enqueues the
+ * same job once per instance on upgrade.
  */
 
 import { z } from "zod";
 
+import { clearLexicalBackfillComplete } from "../../../../persistence/checkpoints.js";
 import {
   enqueueMemoryJob,
   type MemoryJobType,
@@ -42,6 +45,14 @@ async function handleBackfillLexicalIndex({
   body = {},
 }: RouteHandlerArgs): Promise<MessagesLexicalBackfillResult> {
   const { force } = MessagesLexicalBackfillParams.parse(body);
+  if (force === true) {
+    // Clear the completion sentinel at enqueue time so `isLexicalBackfillComplete()`
+    // flips false immediately — the read paths fall back to SQLite FTS in the window
+    // between enqueue and the worker claiming the job, instead of serving from the
+    // stale/emptying `messages_lexical` collection the forced rebuild is about to
+    // reset. The handler clears it again when it runs (idempotent).
+    clearLexicalBackfillComplete();
+  }
   const payload: Record<string, unknown> =
     force === true ? { force: true } : {};
   const jobId = enqueueMemoryJob(BACKFILL_LEXICAL_INDEX_JOB, payload);
@@ -60,7 +71,7 @@ export const ROUTES: RouteDefinition[] = [
     handler: handleBackfillLexicalIndex,
     summary: "Enqueue a resumable backfill of messages into the lexical index",
     description:
-      "Enqueues the cursor-checkpointed backfill job that indexes existing messages into the Qdrant lexical (BM25-style) collection in batches. Resumable and idempotent — re-running continues from the last checkpoint. Pass `force: true` to reset the cursor and re-index from the beginning. Not run at daemon startup.",
+      "Enqueues the cursor-checkpointed backfill job that indexes existing messages into the Qdrant lexical (BM25-style) collection in batches. Resumable and idempotent — re-running continues from the last checkpoint. Pass `force: true` to reset the cursor and re-index from the beginning. The same backfill is also auto-enqueued once per instance on upgrade at assistant startup.",
     tags: ["memory"],
     requestBody: MessagesLexicalBackfillParams,
   },

package/src/plugins/defaults/memory/startup.ts

@@ -30,6 +30,7 @@ import { startMemoryJobsWorker } from "../../../persistence/jobs-worker.js";
 import { getLogger } from "../../../util/logger.js";
 import { getWorkspaceDir } from "../../../util/platform.js";
 import { resolveQdrantUrl } from "./embeddings.js";
+import { maybeEnqueueLexicalBackfillOnUpgrade } from "./job-handlers/backfill-lexical-index.js";
 import { sweepConceptPageFrontmatter } from "./v2/frontmatter-sweep.js";
 import {
   maybeRebuildMemoryV2Concepts,
@@ -206,6 +207,14 @@ export async function runMemoryStartup(config: AssistantConfig): Promise<void> {
   registerMemoryJobHandlers();
   startMemoryJobsWorker();
 
+  // One-time, self-healing backfill of existing messages into the Qdrant
+  // lexical index (`messages_lexical`) on upgrade, so a later read-path flip to
+  // the lexical backend never opens onto an empty index. Enqueue-only and
+  // checkpoint-guarded — the indexing runs off the event loop via the memory
+  // worker started just above; see the function's docstring for the guards and
+  // the deliberate exception it makes to the "no work at daemon startup" rule.
+  maybeEnqueueLexicalBackfillOnUpgrade();
+
   // Seed capability graph nodes (new memory graph system)
   try {
     const { seedCliGraphNodes } = await import("./graph/capability-seed.js");

package/src/runtime/migrations/origin-mode.ts

@@ -7,15 +7,15 @@
  *   - `hasManagedProxyPrereqs()` — true when the daemon has the credentials
  *     it needs to act as a managed-proxy client, i.e. this is a managed
  *     deployment.
- *   - `getDaemonRuntimeMode()` — `"docker"` vs `"bare-metal"`, identifying
- *     where the daemon process is running.
+ *   - `getIsContainerized()` — true when the daemon runs inside a container
+ *     (`IS_CONTAINERIZED`), identifying where the daemon process is running.
  *
  * Folding both into a single helper keeps callers from repeating the
  * combination logic.
  */
 
+import { getIsContainerized } from "../../config/env-registry.js";
 import { hasManagedProxyPrereqs } from "../../providers/platform-proxy/context.js";
-import { getDaemonRuntimeMode } from "../runtime-mode.js";
 
 export type VBundleOriginMode =
   | "managed"
@@ -26,14 +26,14 @@ export type VBundleOriginMode =
  * Returns the origin mode for the current daemon.
  *
  * Managed-proxy prereqs win first (a managed deployment is always
- * "managed" regardless of where the daemon process runs); otherwise docker
- * → "self-hosted-remote", bare-metal → "self-hosted-local".
+ * "managed" regardless of where the daemon process runs); otherwise
+ * containerized → "self-hosted-remote", bare-metal → "self-hosted-local".
  */
 export async function getOriginMode(): Promise<VBundleOriginMode> {
   if (await hasManagedProxyPrereqs()) {
     return "managed";
   }
-  if (getDaemonRuntimeMode() === "docker") {
+  if (getIsContainerized()) {
     return "self-hosted-remote";
   }
   return "self-hosted-local";

package/src/runtime/__tests__/runtime-mode.test.ts

@@ -1,62 +0,0 @@
-/**
- * Tests for `getDaemonRuntimeMode()` — ensures the helper reflects
- * `IS_CONTAINERIZED` environment state via the existing truthy-check
- * semantics shared with `getIsContainerized()`.
- */
-
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-
-import { getDaemonRuntimeMode } from "../runtime-mode.js";
-
-describe("getDaemonRuntimeMode", () => {
-  let savedIsContainerized: string | undefined;
-
-  beforeEach(() => {
-    savedIsContainerized = process.env.IS_CONTAINERIZED;
-  });
-
-  afterEach(() => {
-    if (savedIsContainerized === undefined) {
-      delete process.env.IS_CONTAINERIZED;
-    } else {
-      process.env.IS_CONTAINERIZED = savedIsContainerized;
-    }
-  });
-
-  test("returns 'docker' when IS_CONTAINERIZED=true", () => {
-    process.env.IS_CONTAINERIZED = "true";
-    expect(getDaemonRuntimeMode()).toBe("docker");
-  });
-
-  test("returns 'docker' when IS_CONTAINERIZED=1", () => {
-    // Matches the existing truthy-check in getIsContainerized(), which
-    // treats "1" as equivalent to "true" for boolean env flags.
-    process.env.IS_CONTAINERIZED = "1";
-    expect(getDaemonRuntimeMode()).toBe("docker");
-  });
-
-  test("returns 'bare-metal' when IS_CONTAINERIZED is unset", () => {
-    delete process.env.IS_CONTAINERIZED;
-    expect(getDaemonRuntimeMode()).toBe("bare-metal");
-  });
-
-  test("returns 'bare-metal' when IS_CONTAINERIZED=false", () => {
-    process.env.IS_CONTAINERIZED = "false";
-    expect(getDaemonRuntimeMode()).toBe("bare-metal");
-  });
-
-  test("returns 'bare-metal' when IS_CONTAINERIZED=0", () => {
-    process.env.IS_CONTAINERIZED = "0";
-    expect(getDaemonRuntimeMode()).toBe("bare-metal");
-  });
-
-  test("returns 'bare-metal' when IS_CONTAINERIZED is an empty string", () => {
-    process.env.IS_CONTAINERIZED = "";
-    expect(getDaemonRuntimeMode()).toBe("bare-metal");
-  });
-
-  test("returns 'bare-metal' for arbitrary non-truthy values", () => {
-    process.env.IS_CONTAINERIZED = "yes";
-    expect(getDaemonRuntimeMode()).toBe("bare-metal");
-  });
-});

package/src/runtime/runtime-mode.ts

@@ -1,38 +0,0 @@
-/**
- * Daemon runtime-mode detection.
- *
- * Exposes a single well-named helper `getDaemonRuntimeMode()` that returns
- * `"docker"` when the daemon is running inside a container and
- * `"bare-metal"` otherwise.
- *
- * Under the hood this delegates to `getIsContainerized()` from the env
- * registry, which accepts the standard truthy values for `IS_CONTAINERIZED`
- * (`"true"` or `"1"`). Keeping the check in the registry avoids duplicating
- * the env-parsing semantics across modules.
- *
- * The mode-named API (rather than a boolean) exists to make downstream
- * switch/branch code read naturally — e.g. `if (mode === "docker") { ... }`
- * is clearer than `if (isContainerized) { ... }` when the behavior depends
- * on the specific deployment shape, and it leaves room for additional
- * runtime modes in the future without renaming every callsite.
- *
- * The `DaemonRuntimeMode` type is declared here, alongside the resolver
- * that produces it. `"docker"` describes a daemon running inside a
- * container; `"bare-metal"` describes a daemon running directly on the
- * host.
- */
-
-import { getIsContainerized } from "../config/env-registry.js";
-
-export type DaemonRuntimeMode = "bare-metal" | "docker";
-
-/**
- * Returns the deployment mode the daemon is currently running under.
- *
- * - `"docker"` when `IS_CONTAINERIZED` is set to a truthy value
- *   (`"true"` or `"1"`).
- * - `"bare-metal"` otherwise.
- */
-export function getDaemonRuntimeMode(): DaemonRuntimeMode {
-  return getIsContainerized() ? "docker" : "bare-metal";
-}