@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/persistence/embeddings/embedding-identity.ts

@@ -1,6 +1,7 @@
 import { QdrantClient } from "@qdrant/js-client-rest";
 
 import type { AssistantConfig } from "../../config/types.js";
+import { getLogger } from "../../util/logger.js";
 import {
   resolveBackendDimension,
   selectEmbeddingBackend,
@@ -9,6 +10,8 @@ import { isEmbeddingBillingBreakerOpen } from "./embedding-billing-breaker.js";
 import type { EmbeddingProviderName } from "./embedding-types.js";
 import { resolveQdrantUrl } from "./qdrant-client.js";
 
+const log = getLogger("embedding-identity");
+
 // Inlined rather than imported from `memory/v2/qdrant.ts`: the persistence
 // layer must not import from the memory feature layer (enforced by
 // persistence-layering-guard). The dependency direction is memory → persistence.
@@ -28,6 +31,7 @@ export interface BackendDimensionProbe {
  * reconcile defers rather than committing on a measurement it could not take:
  *   - the billing breaker is open (treated as "backend down");
  *   - no backend is configured/selectable;
+ *   - backend selection throws (e.g. a transient credential-store error);
  *   - the measurement probe throws (provider unreachable).
  *
  * Delegates the dimension measurement to {@link resolveBackendDimension}, the
@@ -40,12 +44,20 @@ export async function probeBackendDimension(
 ): Promise<BackendDimensionProbe | null> {
   if (isEmbeddingBillingBreakerOpen()) return null;
 
-  const { backend } = await selectEmbeddingBackend(config);
-  if (!backend) return null;
+  try {
+    const { backend } = await selectEmbeddingBackend(config);
+    if (!backend) return null;
 
-  const dim = await resolveBackendDimension(backend);
-  if (dim == null) return null;
-  return { provider: backend.provider, model: backend.model, dim };
+    const dim = await resolveBackendDimension(backend);
+    if (dim == null) return null;
+    return { provider: backend.provider, model: backend.model, dim };
+  } catch (err) {
+    // `selectEmbeddingBackend` resolves provider credentials and can reject on a
+    // non-timeout credential-store error; the reconcile relies on a null return
+    // to defer, so honor the never-throws contract and treat it as degraded.
+    log.warn({ err }, "Embedding-dimension probe failed; treating as degraded");
+    return null;
+  }
 }
 
 /**

package/src/persistence/embeddings/embedding-runtime-manager.ts

@@ -571,3 +571,34 @@ export class EmbeddingRuntimeManager {
     }
   }
 }
+
+/**
+ * Download the local embedding runtime in the background (non-blocking).
+ *
+ * Pre-warms the runtime (bun binary + ONNX worker scripts) so the first local
+ * embed doesn't pay the download cost inline. A no-op when the runtime is
+ * already installed. Failures are swallowed with a warning: local embeddings
+ * fall back to cloud backends, so a failed download must never block or crash
+ * daemon startup. Returns immediately; the download proceeds on its own.
+ */
+export function startEmbeddingRuntimeManager(): void {
+  void (async () => {
+    try {
+      const runtimeManager = new EmbeddingRuntimeManager();
+      if (runtimeManager.isReady()) return;
+      log.info("Downloading embedding runtime in background...");
+      await runtimeManager.ensureInstalled();
+      // Reset the sticky local-backend failure flag so auto mode retries
+      // local embeddings without evicting a worker that may already be live.
+      const { resetLocalEmbeddingFailureState } =
+        await import("./embedding-backend.js");
+      resetLocalEmbeddingFailureState();
+      log.info("Embedding runtime download complete");
+    } catch (err) {
+      log.warn(
+        { err },
+        "Embedding runtime download failed — local embeddings will use cloud fallback",
+      );
+    }
+  })();
+}

package/src/persistence/embeddings/messages-lexical-index.ts

@@ -0,0 +1,396 @@
+import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
+import { v5 as uuidv5 } from "uuid";
+
+import { getLogger } from "../../util/logger.js";
+import type { SparseEmbedding } from "./embedding-types.js";
+
+const log = getLogger("messages-lexical-index");
+
+/**
+ * Dedicated Qdrant collection holding the lexical (BM25-style sparse) index for
+ * `messages`. Sparse-only — there is no dense vector. The collection is a
+ * faithful replacement for SQLite FTS5 full-text search over message content,
+ * encoded with the local TF-IDF sparse encoder so no per-message model call is
+ * required to index a message.
+ */
+export const MESSAGES_LEXICAL_COLLECTION = "messages_lexical";
+
+/**
+ * Stable namespace for deriving deterministic Qdrant point ids from message
+ * ids. Using a fixed namespace (instead of `uuid.URL`) keeps point-id identity
+ * scoped to this index, so the same `messageId` always maps to the same point
+ * and re-indexing is idempotent without a pre-read.
+ */
+const MESSAGE_POINT_NAMESPACE = "b6d0d3e2-1f1a-4c3e-9a7c-0e2f5a8d4c11";
+
+export interface MessagesLexicalIndexConfig {
+  url: string;
+  collection?: string;
+  onDisk?: boolean;
+}
+
+export interface MessageLexicalPayload {
+  messageId: string;
+  conversationId: string;
+  createdAt: number;
+}
+
+export interface MessageLexicalSearchResult {
+  messageId: string;
+  score: number;
+}
+
+let _instance: MessagesLexicalIndex | null = null;
+
+export function getMessagesLexicalIndex(): MessagesLexicalIndex {
+  if (!_instance) {
+    throw new Error(
+      "Messages lexical index not initialized. Call initMessagesLexicalIndex() first.",
+    );
+  }
+  return _instance;
+}
+
+export function initMessagesLexicalIndex(
+  config: MessagesLexicalIndexConfig,
+): MessagesLexicalIndex {
+  _instance = new MessagesLexicalIndex(config);
+  return _instance;
+}
+
+/**
+ * Derive the deterministic Qdrant point id for a message. Idempotent:
+ * re-indexing the same `messageId` overwrites the same point.
+ */
+export function messagePointId(messageId: string): string {
+  return uuidv5(messageId, MESSAGE_POINT_NAMESPACE);
+}
+
+export class MessagesLexicalIndex {
+  private readonly client: QdrantRestClient;
+  private readonly collection: string;
+  private readonly onDisk: boolean;
+  private collectionReady = false;
+
+  constructor(config: MessagesLexicalIndexConfig) {
+    this.client = new QdrantRestClient({
+      url: config.url,
+      checkCompatibility: false,
+    });
+    this.collection = config.collection ?? MESSAGES_LEXICAL_COLLECTION;
+    this.onDisk = config.onDisk ?? true;
+  }
+
+  async ensureCollection(): Promise<void> {
+    if (this.collectionReady) return;
+
+    try {
+      const exists = await this.client.collectionExists(this.collection);
+      if (exists.exists) {
+        if (await this.ensurePayloadIndexesSafe()) {
+          this.collectionReady = true;
+        }
+        return;
+      }
+    } catch {
+      // Existence check failed — fall through to creation.
+    }
+
+    log.info(
+      { collection: this.collection },
+      "Creating sparse-only Qdrant collection for messages lexical index",
+    );
+
+    try {
+      await this.client.createCollection(this.collection, {
+        // Sparse-only: no dense `vectors` config. The lexical index is a
+        // BM25-style replacement for FTS5 and never holds a dense vector.
+        sparse_vectors: {
+          sparse: {}, // Qdrant auto-infers sparse vector params
+        },
+        on_disk_payload: this.onDisk,
+      });
+    } catch (err) {
+      // 409 = a concurrent caller created the collection first — that's fine.
+      if (
+        err instanceof Error &&
+        "status" in err &&
+        (err as { status: number }).status === 409
+      ) {
+        if (await this.ensurePayloadIndexesSafe()) {
+          this.collectionReady = true;
+        }
+        return;
+      }
+      throw err;
+    }
+
+    if (await this.ensurePayloadIndexesSafe()) {
+      this.collectionReady = true;
+      log.info(
+        { collection: this.collection },
+        "Messages lexical index collection created with payload indexes",
+      );
+    }
+  }
+
+  async upsertMessage(
+    messageId: string,
+    sparse: SparseEmbedding,
+    payload: { conversationId: string; createdAt: number },
+  ): Promise<void> {
+    await this.upsertMessagesBatch([
+      {
+        messageId,
+        sparse,
+        conversationId: payload.conversationId,
+        createdAt: payload.createdAt,
+      },
+    ]);
+  }
+
+  async upsertMessagesBatch(
+    points: Array<{
+      messageId: string;
+      sparse: SparseEmbedding;
+      conversationId: string;
+      createdAt: number;
+    }>,
+  ): Promise<void> {
+    await this.ensureCollection();
+
+    if (points.length === 0) return;
+
+    const qdrantPoints = points.map((p) => ({
+      id: messagePointId(p.messageId),
+      vector: {
+        sparse: {
+          indices: p.sparse.indices,
+          values: p.sparse.values,
+        },
+      },
+      payload: {
+        message_id: p.messageId,
+        conversation_id: p.conversationId,
+        created_at: p.createdAt,
+      },
+    }));
+
+    const doUpsert = () =>
+      this.client.upsert(this.collection, {
+        wait: true,
+        points: qdrantPoints,
+      });
+
+    try {
+      await doUpsert();
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        await doUpsert();
+      } else {
+        throw err;
+      }
+    }
+  }
+
+  async searchLexical(
+    sparse: SparseEmbedding,
+    limit: number,
+    opts?: { conversationId?: string },
+  ): Promise<MessageLexicalSearchResult[]> {
+    await this.ensureCollection();
+
+    const filter = opts?.conversationId
+      ? {
+          must: [
+            {
+              key: "conversation_id",
+              match: { value: opts.conversationId },
+            },
+          ],
+        }
+      : undefined;
+
+    const queryParams = {
+      query: {
+        indices: sparse.indices,
+        values: sparse.values,
+      },
+      using: "sparse",
+      limit,
+      with_payload: true,
+      filter: filter as Record<string, unknown> | undefined,
+    };
+
+    let results;
+    try {
+      results = await this.client.query(this.collection, queryParams);
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        results = await this.client.query(this.collection, queryParams);
+      } else {
+        throw err;
+      }
+    }
+
+    return (results.points ?? []).map((point) => ({
+      messageId: String(
+        (point.payload as Record<string, unknown> | undefined)?.message_id ??
+          "",
+      ),
+      score: point.score ?? 0,
+    }));
+  }
+
+  async deleteByMessageId(messageId: string): Promise<void> {
+    await this.ensureCollection();
+
+    // Delete by the deterministic point id rather than a `message_id` payload
+    // filter: `message_id` has no payload index, so a filter delete would scan
+    // the whole collection. The point id is recoverable from the message id.
+    const doDelete = () =>
+      this.client.delete(this.collection, {
+        wait: true,
+        points: [messagePointId(messageId)],
+      });
+
+    try {
+      await doDelete();
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        await doDelete();
+      } else {
+        throw err;
+      }
+    }
+  }
+
+  async deleteByConversation(conversationId: string): Promise<void> {
+    await this.ensureCollection();
+
+    const doDelete = () =>
+      this.client.delete(this.collection, {
+        wait: true,
+        filter: {
+          must: [{ key: "conversation_id", match: { value: conversationId } }],
+        },
+      });
+
+    try {
+      await doDelete();
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        await doDelete();
+      } else {
+        throw err;
+      }
+    }
+  }
+
+  async count(): Promise<number> {
+    await this.ensureCollection();
+
+    try {
+      const result = await this.client.count(this.collection, { exact: false });
+      return result.count;
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        const result = await this.client.count(this.collection, {
+          exact: false,
+        });
+        return result.count;
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * Remove every point from the index by dropping the whole collection. Used by
+   * "clear all conversations" — a bulk wipe with no conversation ids left to
+   * enqueue per-conversation purges against. Resetting `collectionReady` means
+   * the next write self-heals via `ensureCollection`. Returns `false` (rather
+   * than throwing) if the collection is absent or the drop fails, mirroring the
+   * shared Qdrant client's `deleteCollection` so callers can treat it as
+   * best-effort.
+   */
+  async clear(): Promise<boolean> {
+    try {
+      const exists = await this.client.collectionExists(this.collection);
+      if (!exists.exists) {
+        this.collectionReady = false;
+        return false;
+      }
+      await this.client.deleteCollection(this.collection);
+      this.collectionReady = false;
+      return true;
+    } catch (err) {
+      log.warn(
+        { err, collection: this.collection },
+        "Failed to clear messages lexical index collection",
+      );
+      return false;
+    }
+  }
+
+  /**
+   * Detect "collection not found" errors from Qdrant so callers can reset
+   * collectionReady and retry after an external deletion.
+   */
+  private isCollectionMissing(err: unknown): boolean {
+    if (
+      err &&
+      typeof err === "object" &&
+      "status" in err &&
+      (err as { status: number }).status === 404
+    ) {
+      return true;
+    }
+    const msg = err instanceof Error ? err.message : String(err);
+    return (
+      msg.includes("Not found") ||
+      msg.includes("doesn't exist") ||
+      msg.includes("not found")
+    );
+  }
+
+  /**
+   * Wraps ensurePayloadIndexes so that a 404 (collection deleted between the
+   * existence check and index creation) resets collectionReady instead of
+   * propagating — the next operation self-heals via ensureCollection.
+   */
+  private async ensurePayloadIndexesSafe(): Promise<boolean> {
+    try {
+      await this.ensurePayloadIndexes();
+      return true;
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        return false;
+      }
+      throw err;
+    }
+  }
+
+  private async ensurePayloadIndexes(): Promise<void> {
+    await Promise.all([
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "conversation_id",
+        field_schema: "keyword",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "created_at",
+        field_schema: "integer",
+      }),
+    ]);
+  }
+}

package/src/persistence/embeddings/qdrant-client.ts

@@ -1,9 +1,14 @@
+import { existsSync } from "node:fs";
+import { mkdir, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
 import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
 import { v4 as uuid } from "uuid";
 
 import { getQdrantHttpPortEnv, getQdrantUrlEnv } from "../../config/env.js";
 import type { AssistantConfig } from "../../config/types.js";
 import { getLogger } from "../../util/logger.js";
+import { getDataDir } from "../../util/platform.js";
 
 const log = getLogger("qdrant-client");
 
@@ -85,6 +90,57 @@ export function stripLegacySparseSuffix(sentinel: string): string {
   return sentinel.replace(/:sparse-v\d+$/, "");
 }
 
+/**
+ * Marker file written before the v1 collection's destructive dimension/model
+ * recreate, cleared by the startup lifecycle hook once the `rebuild_index` job
+ * has been enqueued.
+ *
+ * Closes a data-loss window in {@link VellumQdrantClient.ensureCollection}: the
+ * `migrated` signal that tells the lifecycle hook to enqueue `rebuild_index`
+ * otherwise lives only in a local variable returned after `deleteCollection` +
+ * `createCollection` both succeed. If Qdrant fails between the delete and the
+ * return — or the process crashes — that signal is lost: the next startup finds
+ * the collection missing, recreates it empty, reports `migrated: false`, and
+ * the v1 vectors stay dropped until each row is individually re-touched.
+ * Persisting the intent on disk *before* the delete lets it survive crashes and
+ * transient failures: every later `ensureCollection` reports `migrated: true`
+ * until the hook enqueues the rebuild and clears the sentinel. Mirrors the v2
+ * reembed sentinel in `plugins/defaults/memory/v2/qdrant.ts`.
+ *
+ * Distinct from the in-collection model-identity sentinel point handled by
+ * {@link VellumQdrantClient.readSentinel} / `writeSentinel` — that records the
+ * embedding model; this records "a rebuild is owed".
+ */
+const REBUILD_SENTINEL_FILENAME = ".memory-v1-rebuild-required";
+
+function rebuildSentinelPath(): string {
+  return join(getDataDir(), REBUILD_SENTINEL_FILENAME);
+}
+
+function rebuildSentinelExists(): boolean {
+  return existsSync(rebuildSentinelPath());
+}
+
+async function writeRebuildSentinel(): Promise<void> {
+  const path = rebuildSentinelPath();
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, "");
+}
+
+/**
+ * Remove the rebuild sentinel after the startup lifecycle hook has enqueued the
+ * `rebuild_index` job. Idempotent — a missing file is not an error.
+ */
+export async function clearRebuildSentinel(): Promise<void> {
+  try {
+    await unlink(rebuildSentinelPath());
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw err;
+    }
+  }
+}
+
 let _instance: VellumQdrantClient | null = null;
 
 export function getQdrantClient(): VellumQdrantClient {
@@ -129,7 +185,11 @@ export class VellumQdrantClient {
   async ensureCollection(): Promise<{ migrated: boolean }> {
     if (this.collectionReady) return { migrated: false };
 
-    let migrated = false;
+    // A leftover sentinel means a prior boot deleted the collection but never
+    // got to enqueue the rebuild (createCollection threw, or the process died
+    // before the lifecycle hook ran). Carry that signal forward until the hook
+    // clears it, even when the collection now looks freshly created or intact.
+    let migrated = rebuildSentinelExists();
 
     try {
       const exists = await this.client.collectionExists(this.collection);
@@ -186,7 +246,7 @@ export class VellumQdrantClient {
               },
               "Qdrant collection uses unnamed vectors (legacy) — deleting and recreating with named vectors. Embeddings will be re-indexed.",
             );
-            await this.client.deleteCollection(this.collection);
+            await this.deleteCollectionForRebuild();
             migrated = true;
             // Fall through to collection creation below
           } else if (dimMismatch || modelMismatch) {
@@ -207,7 +267,7 @@ export class VellumQdrantClient {
               },
               "Qdrant collection incompatible (dimension/model change) — deleting and recreating. Embeddings will be regenerated on demand.",
             );
-            await this.client.deleteCollection(this.collection);
+            await this.deleteCollectionForRebuild();
             migrated = true;
             // Fall through to collection creation below
           } else {
@@ -217,7 +277,11 @@ export class VellumQdrantClient {
             if (needsSentinelRewrite && this.embeddingModel) {
               await this.writeSentinel(this.embeddingModel);
             }
-            return { migrated: false };
+            // `migrated` is false in steady state; it is only true here when a
+            // prior recreate wrote the sentinel but never cleared it, in which
+            // case the rebuild is still owed against this (possibly empty)
+            // collection.
+            return { migrated };
           }
         } catch (err) {
           log.warn(
@@ -227,7 +291,7 @@ export class VellumQdrantClient {
           if (await this.ensurePayloadIndexesSafe()) {
             this.collectionReady = true;
           }
-          return { migrated: false };
+          return { migrated };
         }
       }
     } catch {
@@ -302,6 +366,18 @@ export class VellumQdrantClient {
     return { migrated };
   }
 
+  /**
+   * Delete the collection as the first half of a destructive dimension/model
+   * recreate. Persists the rebuild sentinel BEFORE the delete so a crash or
+   * transient Qdrant failure between the delete and the recreate still triggers
+   * `rebuild_index` on the next ensure call. Callers set `migrated = true` and
+   * fall through to recreation.
+   */
+  private async deleteCollectionForRebuild(): Promise<void> {
+    await writeRebuildSentinel();
+    await this.client.deleteCollection(this.collection);
+  }
+
   async upsert(
     targetType: QdrantPointPayload["target_type"],
     targetId: string,

package/src/persistence/group-crud.ts

@@ -37,6 +37,7 @@ export function listGroups(): ConversationGroupRow[] {
     created_at: number;
     updated_at: number;
   }>(
+    "group:listGroups",
     "SELECT id, name, sort_position, is_system_group, created_at, updated_at FROM conversation_groups WHERE id NOT GLOB '_*' ORDER BY sort_position ASC",
   );
   return rows.map((r) => ({
@@ -63,6 +64,7 @@ export function getGroup(groupId: string): ConversationGroupRow | null {
     created_at: number;
     updated_at: number;
   }>(
+    "group:getGroup",
     "SELECT id, name, sort_position, is_system_group, created_at, updated_at FROM conversation_groups WHERE id = ?",
     groupId,
   );
@@ -91,12 +93,14 @@ export function createGroup(name: string): ConversationGroupRow {
   ensureGroupMigration();
   const maxPos =
     rawGet<{ max: number | null }>(
+      "group:createGroup:maxPos",
       "SELECT MAX(sort_position) as max FROM conversation_groups WHERE is_system_group = 0",
     )?.max ?? 3;
   const sortPosition = maxPos + 1;
   const id = uuid();
   const now = Math.floor(Date.now() / 1000);
   rawRun(
+    "group:createGroup:insert",
     "INSERT INTO conversation_groups (id, name, sort_position, is_system_group, created_at, updated_at) VALUES (?, ?, ?, 0, ?, ?)",
     id,
     name,
@@ -146,6 +150,7 @@ export function updateGroup(
   values.push(groupId);
 
   rawRun(
+    "group:updateGroup",
     `UPDATE conversation_groups SET ${fields.join(", ")} WHERE id = ?`,
     ...values,
   );
@@ -164,10 +169,15 @@ export function updateGroup(
 export function deleteGroup(groupId: string): boolean {
   ensureGroupMigration();
   rawRun(
+    "group:deleteGroup:reassign",
     "UPDATE conversations SET group_id = 'system:all' WHERE group_id = ?",
     groupId,
   );
-  rawRun("DELETE FROM conversation_groups WHERE id = ?", groupId);
+  rawRun(
+    "group:deleteGroup:delete",
+    "DELETE FROM conversation_groups WHERE id = ?",
+    groupId,
+  );
   return true;
 }
 
@@ -185,6 +195,7 @@ export function reorderGroups(
     for (const update of updates) {
       // Look up the group first — skip unknown/stale IDs and system groups
       const group = rawGet<{ id: string; is_system_group: number }>(
+        "group:reorderGroups:select",
         "SELECT id, is_system_group FROM conversation_groups WHERE id = ?",
         update.groupId,
       );
@@ -197,6 +208,7 @@ export function reorderGroups(
         );
       }
       rawRun(
+        "group:reorderGroups:update",
         "UPDATE conversation_groups SET sort_position = ?, updated_at = ? WHERE id = ?",
         update.sortPosition,
         now,