@vellumai/assistant 0.4.44 → 0.4.45

package/src/memory/qdrant-client.ts

@@ -11,10 +11,11 @@ export interface QdrantClientConfig {
   vectorSize: number;
   onDisk: boolean;
   quantization: "scalar" | "none";
+  embeddingModel?: string;
 }
 
 export interface QdrantPointPayload {
-  target_type: "segment" | "item" | "summary";
+  target_type: "segment" | "item" | "summary" | "media";
   target_id: string;
   text: string;
   kind?: string;
@@ -26,7 +27,9 @@ export interface QdrantPointPayload {
   last_seen_at?: number;
   conversation_id?: string;
   message_id?: string;
+  memory_scope_id?: string;
   entity_ids?: string[];
+  modality?: "text" | "image" | "audio" | "video";
 }
 
 export interface QdrantSearchResult {
@@ -59,8 +62,11 @@ export class VellumQdrantClient {
   private readonly vectorSize: number;
   private readonly onDisk: boolean;
   private readonly quantization: "scalar" | "none";
+  private readonly embeddingModel?: string;
   private collectionReady = false;
 
+  private readonly SENTINEL_ID = "00000000-0000-0000-0000-000000000000";
+
   constructor(config: QdrantClientConfig) {
     this.client = new QdrantRestClient({
       url: config.url,
@@ -70,6 +76,7 @@ export class VellumQdrantClient {
     this.vectorSize = config.vectorSize;
     this.onDisk = config.onDisk;
     this.quantization = config.quantization;
+    this.embeddingModel = config.embeddingModel;
   }
 
   async ensureCollection(): Promise<void> {
@@ -78,8 +85,51 @@ export class VellumQdrantClient {
     try {
       const exists = await this.client.collectionExists(this.collection);
       if (exists.exists) {
-        this.collectionReady = true;
-        return;
+        try {
+          const info = await this.client.getCollection(this.collection);
+          const currentSize = (
+            info.config?.params?.vectors as { size?: number }
+          )?.size;
+          const dimMismatch =
+            currentSize != null && currentSize !== this.vectorSize;
+
+          // Check model identity via a sentinel point that stores the embedding model
+          let modelMismatch = false;
+          if (this.embeddingModel) {
+            const sentinel = await this.readSentinel();
+            if (sentinel && sentinel !== this.embeddingModel) {
+              modelMismatch = true;
+            }
+          }
+
+          if (dimMismatch || modelMismatch) {
+            log.warn(
+              {
+                collection: this.collection,
+                currentSize,
+                expectedSize: this.vectorSize,
+                modelMismatch,
+              },
+              "Qdrant collection incompatible (dimension or model change) — deleting and recreating. Embeddings will be regenerated on demand.",
+            );
+            await this.client.deleteCollection(this.collection);
+            // Fall through to collection creation below
+          } else {
+            if (await this.ensurePayloadIndexesSafe()) {
+              this.collectionReady = true;
+            }
+            return;
+          }
+        } catch (err) {
+          log.warn(
+            { err },
+            "Failed to verify collection compatibility, assuming compatible",
+          );
+          if (await this.ensurePayloadIndexesSafe()) {
+            this.collectionReady = true;
+          }
+          return;
+        }
       }
     } catch {
       // Collection doesn't exist, create it
@@ -121,49 +171,30 @@ export class VellumQdrantClient {
         "status" in err &&
         (err as { status: number }).status === 409
       ) {
-        this.collectionReady = true;
+        if (await this.ensurePayloadIndexesSafe()) {
+          this.collectionReady = true;
+        }
         return;
       }
       throw err;
     }
 
-    // Create payload indexes for efficient filtering
-    await Promise.all([
-      this.client.createPayloadIndex(this.collection, {
-        field_name: "target_type",
-        field_schema: "keyword",
-      }),
-      this.client.createPayloadIndex(this.collection, {
-        field_name: "target_id",
-        field_schema: "keyword",
-      }),
-      this.client.createPayloadIndex(this.collection, {
-        field_name: "kind",
-        field_schema: "keyword",
-      }),
-      this.client.createPayloadIndex(this.collection, {
-        field_name: "status",
-        field_schema: "keyword",
-      }),
-      this.client.createPayloadIndex(this.collection, {
-        field_name: "created_at",
-        field_schema: "integer",
-      }),
-      this.client.createPayloadIndex(this.collection, {
-        field_name: "conversation_id",
-        field_schema: "keyword",
-      }),
-    ]);
+    if (await this.ensurePayloadIndexesSafe()) {
+      // Write sentinel point to record the active embedding model
+      if (this.embeddingModel) {
+        await this.writeSentinel(this.embeddingModel);
+      }
 
-    this.collectionReady = true;
-    log.info(
-      { collection: this.collection },
-      "Qdrant collection created with payload indexes",
-    );
+      this.collectionReady = true;
+      log.info(
+        { collection: this.collection },
+        "Qdrant collection created with payload indexes",
+      );
+    }
   }
 
   async upsert(
-    targetType: "segment" | "item" | "summary",
+    targetType: "segment" | "item" | "summary" | "media",
     targetId: string,
     vector: number[],
     payload: Omit<QdrantPointPayload, "target_type" | "target_id">,
@@ -257,7 +288,7 @@ export class VellumQdrantClient {
   async searchWithFilter(
     vector: number[],
     limit: number,
-    targetTypes: Array<"segment" | "item" | "summary">,
+    targetTypes: Array<"segment" | "item" | "summary" | "media">,
     excludeMessageIds?: string[],
   ): Promise<QdrantSearchResult[]> {
     const mustConditions: Array<Record<string, unknown>> = [
@@ -277,12 +308,17 @@ export class VellumQdrantClient {
               { key: "status", match: { value: "active" } },
             ],
           },
-          { key: "target_type", match: { any: ["segment", "summary"] } },
+          {
+            key: "target_type",
+            match: { any: ["segment", "summary", "media"] },
+          },
         ],
       });
     }
 
-    const mustNotConditions: Array<Record<string, unknown>> = [];
+    const mustNotConditions: Array<Record<string, unknown>> = [
+      { key: "_meta", match: { value: true } },
+    ];
     if (excludeMessageIds && excludeMessageIds.length > 0) {
       mustNotConditions.push({
         key: "message_id",
@@ -292,10 +328,8 @@ export class VellumQdrantClient {
 
     const filter: Record<string, unknown> = {
       must: mustConditions,
+      must_not: mustNotConditions,
     };
-    if (mustNotConditions.length > 0) {
-      filter.must_not = mustNotConditions;
-    }
 
     return this.search(vector, limit, filter);
   }
@@ -384,6 +418,87 @@ export class VellumQdrantClient {
     );
   }
 
+  /**
+   * Wraps ensurePayloadIndexes so that a 404 (collection deleted between
+   * our existence check and index creation) resets collectionReady instead
+   * of propagating — the next operation will self-heal via ensureCollection.
+   */
+  private async ensurePayloadIndexesSafe(): Promise<boolean> {
+    try {
+      await this.ensurePayloadIndexes();
+      return true;
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        return false;
+      }
+      throw err;
+    }
+  }
+
+  private async ensurePayloadIndexes(): Promise<void> {
+    await Promise.all([
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "target_type",
+        field_schema: "keyword",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "target_id",
+        field_schema: "keyword",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "kind",
+        field_schema: "keyword",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "status",
+        field_schema: "keyword",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "created_at",
+        field_schema: "integer",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "conversation_id",
+        field_schema: "keyword",
+      }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "modality",
+        field_schema: "keyword",
+      }),
+    ]);
+  }
+
+  private async readSentinel(): Promise<string | null> {
+    try {
+      const points = await this.client.retrieve(this.collection, {
+        ids: [this.SENTINEL_ID],
+        with_payload: true,
+        with_vector: false,
+      });
+      if (points.length === 0) return null;
+      return (
+        ((points[0].payload as Record<string, unknown>)
+          ?.embedding_model as string) ?? null
+      );
+    } catch {
+      return null;
+    }
+  }
+
+  private async writeSentinel(model: string): Promise<void> {
+    await this.client.upsert(this.collection, {
+      wait: true,
+      points: [
+        {
+          id: this.SENTINEL_ID,
+          vector: new Array(this.vectorSize).fill(0), // zero vector, never matched in search
+          payload: { _meta: true, embedding_model: model },
+        },
+      ],
+    });
+  }
+
   private async findByTarget(
     targetType: string,
     targetId: string,

package/src/memory/retriever.test.ts

@@ -26,7 +26,6 @@ mock.module("../util/platform.js", () => ({
   isMacOS: () => process.platform === "darwin",
   isLinux: () => process.platform === "linux",
   isWindows: () => process.platform === "win32",
-  getSocketPath: () => join(testDir, "test.sock"),
   getPidPath: () => join(testDir, "test.pid"),
   getDbPath: () => join(testDir, "test.db"),
   getLogPath: () => join(testDir, "test.log"),

package/src/memory/retriever.ts

@@ -538,7 +538,7 @@ async function generateQueryEmbedding(
         };
       }
       logMemoryEmbeddingWarning(err, "query");
-      degraded = config.memory.embeddings.required;
+      degraded = true;
       reason = `memory.embedding_failure: ${
         err instanceof Error ? err.message : String(err)
       }`;
@@ -828,7 +828,17 @@ export async function buildMemoryRecall(
   // degradation state. This ensures results computed with boosted limits
   // are marked degraded and excluded from the recall cache — preventing
   // stale boosted results from being served after the breaker closes.
-  if (collected.semanticSearchFailed || collected.semanticUnavailable) {
+  //
+  // Exception: when semanticUnavailable is solely because no embedding
+  // provider is configured (queryVector == null) and embeddings are not
+  // required, lexical-only results are the expected steady state — do not
+  // mark as degraded.
+  const semanticActuallyFailed =
+    collected.semanticSearchFailed ||
+    (collected.semanticUnavailable &&
+      (embeddingResult.queryVector != null ||
+        config.memory.embeddings.required));
+  if (semanticActuallyFailed) {
     embeddingResult.degraded = true;
     embeddingResult.reason =
       embeddingResult.reason ??

package/src/memory/schema/infrastructure.ts

@@ -7,39 +7,11 @@ import {
   text,
 } from "drizzle-orm/sqlite-core";
 
-export const accounts = sqliteTable("accounts", {
-  id: text("id").primaryKey(),
-  service: text("service").notNull(),
-  username: text("username"),
-  email: text("email"),
-  displayName: text("display_name"),
-  status: text("status").notNull().default("active"),
-  credentialRef: text("credential_ref"),
-  metadataJson: text("metadata_json"),
-  createdAt: integer("created_at").notNull(),
-  updatedAt: integer("updated_at").notNull(),
-});
-
-export const reminders = sqliteTable("reminders", {
-  id: text("id").primaryKey(),
-  label: text("label").notNull(),
-  message: text("message").notNull(),
-  fireAt: integer("fire_at").notNull(), // epoch ms, absolute timestamp
-  mode: text("mode").notNull(), // 'notify' | 'execute'
-  status: text("status").notNull(), // 'pending' | 'firing' | 'fired' | 'cancelled'
-  firedAt: integer("fired_at"),
-  conversationId: text("conversation_id"),
-  routingIntent: text("routing_intent").notNull().default("all_channels"), // 'single_channel' | 'multi_channel' | 'all_channels'
-  routingHintsJson: text("routing_hints_json").notNull().default("{}"),
-  createdAt: integer("created_at").notNull(),
-  updatedAt: integer("updated_at").notNull(),
-});
-
 export const cronJobs = sqliteTable("cron_jobs", {
   id: text("id").primaryKey(),
   name: text("name").notNull(),
   enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
-  cronExpression: text("cron_expression").notNull(), // e.g. '0 9 * * 1-5'
+  cronExpression: text("cron_expression"), // nullable for one-shot schedules; e.g. '0 9 * * 1-5'
   scheduleSyntax: text("schedule_syntax").notNull().default("cron"), // 'cron' | 'rrule'
   timezone: text("timezone"), // e.g. 'America/Los_Angeles'
   message: text("message").notNull(),
@@ -48,6 +20,10 @@ export const cronJobs = sqliteTable("cron_jobs", {
   lastStatus: text("last_status"), // 'ok' | 'error'
   retryCount: integer("retry_count").notNull().default(0),
   createdBy: text("created_by").notNull(), // 'agent' | 'user'
+  mode: text("mode").notNull().default("execute"), // 'notify' | 'execute'
+  routingIntent: text("routing_intent").notNull().default("all_channels"), // 'single_channel' | 'multi_channel' | 'all_channels'
+  routingHintsJson: text("routing_hints_json").notNull().default("{}"),
+  status: text("status").notNull().default("active"), // 'active' | 'firing' | 'fired' | 'cancelled'
   createdAt: integer("created_at").notNull(),
   updatedAt: integer("updated_at").notNull(),
 });

package/src/memory/search/formatting.ts

@@ -99,15 +99,25 @@ function buildStructuredInjectedText(candidates: Candidate[]): string {
   for (const candidate of ordered) {
     const absolute = formatAbsoluteTime(candidate.createdAt);
     const relative = formatRelativeTime(candidate.createdAt);
-    parts.push(
-      `<entry kind="${escapeXmlAttr(candidate.kind)}" type="${
-        candidate.type
-      }" confidence="${candidate.confidence.toFixed(
-        2,
-      )}" time="${absolute} (${relative})">` +
-        escapeXmlTags(truncate(candidate.text, 320)) +
-        "</entry>",
-    );
+    if (candidate.type === "media") {
+      const modality = candidate.modality ?? "media";
+      const subject = candidate.kind !== "media" ? ` (${candidate.kind})` : "";
+      parts.push(
+        `<entry kind="${escapeXmlAttr(candidate.kind)}" type="media" confidence="${candidate.confidence.toFixed(
+          2,
+        )}" time="${absolute} (${relative})">[Recalled ${modality}${subject}]</entry>`,
+      );
+    } else {
+      parts.push(
+        `<entry kind="${escapeXmlAttr(candidate.kind)}" type="${
+          candidate.type
+        }" confidence="${candidate.confidence.toFixed(
+          2,
+        )}" time="${absolute} (${relative})">` +
+          escapeXmlTags(truncate(candidate.text, 320)) +
+          "</entry>",
+      );
+    }
   }
   parts.push("</entries>");
   parts.push(MEMORY_RECALL_CLOSE_TAG);
@@ -146,6 +156,9 @@ export function applyAttentionOrdering(candidates: Candidate[]): Candidate[] {
 }
 
 function formatCandidateLine(candidate: Candidate): string {
+  if (candidate.type === "media") {
+    return formatMediaCandidateLine(candidate);
+  }
   const absolute = formatAbsoluteTime(candidate.createdAt);
   const relative = formatRelativeTime(candidate.createdAt);
   return `- <kind>${candidate.kind}</kind> ${escapeXmlTags(
@@ -153,6 +166,18 @@ function formatCandidateLine(candidate: Candidate): string {
   )} (${absolute} \u00b7 ${relative})`;
 }
 
+/**
+ * Format a media candidate as a descriptive reference. Since the LLM can't
+ * see the actual image/audio from memory recall text, we provide a reference
+ * that gives awareness of relevant media in memory.
+ */
+function formatMediaCandidateLine(candidate: Candidate): string {
+  const modality = candidate.modality ?? "media";
+  const subject = candidate.kind !== "media" ? ` (${candidate.kind})` : "";
+  const relative = formatRelativeTime(candidate.createdAt);
+  return `- [Recalled ${modality}${subject} from ${relative}]`;
+}
+
 /**
  * Escape XML-like tag sequences in recalled text to prevent delimiter injection.
  * Recalled content is interpolated verbatim inside `<memory>` wrapper tags,

package/src/memory/search/semantic.ts

@@ -10,6 +10,7 @@ import {
 import type { QdrantSearchResult } from "../qdrant-client.js";
 import { getQdrantClient } from "../qdrant-client.js";
 import {
+  conversations,
   memoryItems,
   memoryItemSources,
   memorySegments,
@@ -43,7 +44,7 @@ export async function semanticSearch(
     qdrant.searchWithFilter(
       queryVector,
       fetchLimit,
-      ["item", "summary", "segment"],
+      ["item", "summary", "segment", "media"],
       excludedMessageIds,
     ),
   );
@@ -54,12 +55,16 @@ export async function semanticSearch(
   const itemTargetIds: string[] = [];
   const summaryTargetIds: string[] = [];
   const segmentTargetIds: string[] = [];
+  const mediaConversationIds: string[] = [];
   for (const r of results) {
     if (r.payload.target_type === "item")
       itemTargetIds.push(r.payload.target_id);
     else if (r.payload.target_type === "summary")
       summaryTargetIds.push(r.payload.target_id);
-    else segmentTargetIds.push(r.payload.target_id);
+    else if (r.payload.target_type === "segment")
+      segmentTargetIds.push(r.payload.target_id);
+    else if (r.payload.target_type === "media" && r.payload.conversation_id)
+      mediaConversationIds.push(r.payload.conversation_id);
   }
 
   const itemsMap = new Map<string, typeof memoryItems.$inferSelect>();
@@ -109,6 +114,23 @@ export async function semanticSearch(
     for (const seg of allSegments) segmentsMap.set(seg.id, seg);
   }
 
+  // Batch-fetch conversation scope IDs for media results to avoid N+1 queries.
+  // When a conversation is not found (deleted), its media is excluded rather than
+  // falling back to "default" scope, which would leak private media.
+  const mediaScopeMap = new Map<string, string>();
+  if (scopeIds && mediaConversationIds.length > 0) {
+    const unique = [...new Set(mediaConversationIds)];
+    const rows = db
+      .select({
+        id: conversations.id,
+        memoryScopeId: conversations.memoryScopeId,
+      })
+      .from(conversations)
+      .where(inArray(conversations.id, unique))
+      .all();
+    for (const row of rows) mediaScopeMap.set(row.id, row.memoryScopeId);
+  }
+
   const excludedSet =
     excludedMessageIds.length > 0 ? new Set(excludedMessageIds) : null;
 
@@ -164,6 +186,39 @@ export async function semanticSearch(
         recency: computeRecencyScore(payload.last_seen_at ?? createdAt),
         finalScore: 0,
       });
+    } else if (payload.target_type === "media") {
+      // Use stored memory_scope_id when available; fall back to deriving
+      // scope from conversation_id for legacy media points.
+      // If the conversation was deleted, skip the media to avoid leaking
+      // private media into the default scope.
+      if (scopeIds) {
+        let mediaScopeId: string | undefined;
+        if (payload.memory_scope_id) {
+          mediaScopeId = payload.memory_scope_id;
+        } else if (payload.conversation_id) {
+          mediaScopeId = mediaScopeMap.get(payload.conversation_id);
+          if (!mediaScopeId) continue; // conversation deleted — skip
+        } else {
+          mediaScopeId = "default";
+        }
+        if (!scopeIds.includes(mediaScopeId)) continue;
+      }
+      candidates.push({
+        key: `media:${payload.target_id}`,
+        type: "media",
+        id: payload.target_id,
+        source: "semantic",
+        text: payload.text,
+        kind: payload.kind ?? "media",
+        modality: payload.modality,
+        confidence: 0.7,
+        importance: 0.6,
+        createdAt,
+        lexical: 0,
+        semantic,
+        recency: computeRecencyScore(createdAt),
+        finalScore: 0,
+      });
     } else {
       if (scopeIds) {
         const segment = segmentsMap.get(payload.target_id);

package/src/memory/search/types.ts

@@ -1,4 +1,4 @@
-export type CandidateType = "segment" | "item" | "summary";
+export type CandidateType = "segment" | "item" | "summary" | "media";
 export type CandidateSource =
   | "lexical"
   | "semantic"
@@ -14,6 +14,7 @@ export interface Candidate {
   source: CandidateSource;
   text: string;
   kind: string;
+  modality?: "text" | "image" | "audio" | "video";
   confidence: number;
   importance: number;
   createdAt: number;

package/src/notifications/AGENTS.md

@@ -1,5 +1,5 @@
 # Notification Pipeline
 
-All notification producers **MUST** go through `emitNotificationSignal()` in `notifications/emit-signal.ts`. Do not bypass the pipeline by broadcasting IPC events directly -- the pipeline handles event persistence, deduplication, decision routing, and delivery audit.
+All notification producers **MUST** go through `emitNotificationSignal()` in `notifications/emit-signal.ts`. Do not bypass the pipeline by broadcasting events directly -- the pipeline handles event persistence, deduplication, decision routing, and delivery audit.
 
-When a notification flow creates a server-side conversation (e.g. guardian question threads, task run threads), the conversation and initial message **MUST** be persisted before the IPC thread-created event is emitted. This ensures the macOS/iOS client can immediately fetch the conversation contents when it receives the event.
+When a notification flow creates a server-side conversation (e.g. guardian question threads, task run threads), the conversation and initial message **MUST** be persisted before the thread-created event is emitted. This ensures the macOS/iOS client can immediately fetch the conversation contents when it receives the event.