@vellumai/assistant 0.4.44 → 0.4.45

package/src/memory/indexer.ts

@@ -1,16 +1,21 @@
 import { createHash } from "crypto";
 import { desc, eq } from "drizzle-orm";
 
+import { getConfig } from "../config/loader.js";
 import type { MemoryConfig } from "../config/types.js";
 import type { TrustClass } from "../runtime/actor-trust-resolver.js";
 import { getLogger } from "../util/logger.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
 import { getDb } from "./db.js";
+import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
 import {
   enqueueMemoryJob,
   enqueueResolvePendingConflictsForMessageJob,
 } from "./jobs-store.js";
-import { extractTextFromStoredMessageContent } from "./message-content.js";
+import {
+  extractMediaBlocks,
+  extractTextFromStoredMessageContent,
+} from "./message-content.js";
 import { bumpMemoryVersion } from "./recall-cache.js";
 import { memorySegments } from "./schema.js";
 import { segmentText } from "./segmenter.js";
@@ -73,6 +78,13 @@ export function indexMessageNow(
   const shouldResolveConflicts =
     input.role === "user" && config.conflicts.enabled;
 
+  // Check if the resolved embedding backend supports multimodal input.
+  // Only enqueue embed_attachment jobs when it does (currently Gemini only).
+  const supportsMultimodal = selectedBackendSupportsMultimodal(getConfig());
+  const mediaBlocks = supportsMultimodal
+    ? extractMediaBlocks(input.content).filter((b) => b.type === "image")
+    : [];
+
   // Wrap all segment inserts and job enqueues in a single transaction so they
   // either all succeed or all roll back, preventing partial/orphaned state.
   let skippedEmbedJobs = 0;
@@ -121,6 +133,17 @@ export function indexMessageNow(
       }
     }
 
+    // Enqueue embed_attachment jobs for image content blocks when the
+    // embedding provider supports multimodal (Gemini only).
+    for (const block of mediaBlocks) {
+      enqueueMemoryJob(
+        "embed_attachment",
+        { messageId: input.messageId, blockIndex: block.index },
+        Date.now(),
+        tx,
+      );
+    }
+
     if (shouldExtract && isTrustedActor) {
       enqueueMemoryJob(
         "extract_items",
@@ -168,6 +191,7 @@ export function indexMessageNow(
   const enqueuedJobs =
     segments.length -
     skippedEmbedJobs +
+    mediaBlocks.length +
     (shouldExtract && !extractionGated ? 2 : 1) +
     (shouldResolveConflicts && !extractionGated ? 1 : 0);
   return {

package/src/memory/job-handlers/embedding.test.ts

@@ -0,0 +1,258 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  mock,
+  test,
+} from "bun:test";
+
+const testDir = mkdtempSync(join(tmpdir(), "embed-media-test-"));
+
+mock.module("../../util/platform.js", () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Track calls to embedAndUpsert
+const embedAndUpsertCalls: Array<{
+  config: unknown;
+  targetType: string;
+  targetId: string;
+  input: unknown;
+  extraPayload: unknown;
+}> = [];
+
+mock.module("../job-utils.js", () => ({
+  asString: (value: unknown) =>
+    typeof value === "string" && value.length > 0 ? value : null,
+  embedAndUpsert: async (
+    config: unknown,
+    targetType: string,
+    targetId: string,
+    input: unknown,
+    extraPayload: unknown,
+  ) => {
+    embedAndUpsertCalls.push({
+      config,
+      targetType,
+      targetId,
+      input,
+      extraPayload,
+    });
+  },
+}));
+
+import { DEFAULT_CONFIG } from "../../config/defaults.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { getDb, initializeDb, resetDb } from "../db.js";
+import type { MemoryJob } from "../jobs-store.js";
+import { mediaAssets } from "../schema.js";
+import { embedMediaJob } from "./embedding.js";
+
+const TEST_CONFIG: AssistantConfig = {
+  ...DEFAULT_CONFIG,
+  memory: {
+    ...DEFAULT_CONFIG.memory,
+    extraction: {
+      ...DEFAULT_CONFIG.memory.extraction,
+      useLLM: false,
+    },
+  },
+};
+
+describe("embedMediaJob", () => {
+  beforeAll(() => {
+    initializeDb();
+  });
+
+  beforeEach(() => {
+    embedAndUpsertCalls.length = 0;
+    resetDb();
+    initializeDb();
+  });
+
+  afterAll(() => {
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  function makeJob(payload: Record<string, unknown>): MemoryJob {
+    return {
+      id: "job-1",
+      type: "embed_media",
+      payload,
+      status: "running",
+      attempts: 0,
+      deferrals: 0,
+      runAfter: 0,
+      lastError: null,
+      startedAt: Date.now(),
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+    };
+  }
+
+  test("skips when assetId is missing", async () => {
+    await embedMediaJob(makeJob({}), TEST_CONFIG);
+    expect(embedAndUpsertCalls).toHaveLength(0);
+  });
+
+  test("skips when asset is not found", async () => {
+    await embedMediaJob(makeJob({ assetId: "nonexistent" }), TEST_CONFIG);
+    expect(embedAndUpsertCalls).toHaveLength(0);
+  });
+
+  test("skips when asset status is not indexed", async () => {
+    const db = getDb();
+    const now = Date.now();
+    // Create a temp file
+    const filePath = join(testDir, "test-registered.png");
+    writeFileSync(filePath, Buffer.from("fake image data"));
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-registered",
+        title: "Test Image",
+        filePath,
+        mimeType: "image/png",
+        durationSeconds: null,
+        fileHash: "hash-registered",
+        status: "registered",
+        mediaType: "image",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(
+      makeJob({ assetId: "asset-registered" }),
+      TEST_CONFIG,
+    );
+    expect(embedAndUpsertCalls).toHaveLength(0);
+  });
+
+  test("embeds indexed image asset with correct input type and target", async () => {
+    const db = getDb();
+    const now = Date.now();
+    const filePath = join(testDir, "test-image.png");
+    const imageData = Buffer.from("fake png data");
+    writeFileSync(filePath, imageData);
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-image",
+        title: "My Screenshot",
+        filePath,
+        mimeType: "image/png",
+        durationSeconds: null,
+        fileHash: "hash-image",
+        status: "indexed",
+        mediaType: "image",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(makeJob({ assetId: "asset-image" }), TEST_CONFIG);
+
+    expect(embedAndUpsertCalls).toHaveLength(1);
+    const call = embedAndUpsertCalls[0];
+    expect(call.targetType).toBe("media");
+    expect(call.targetId).toBe("asset-image");
+    expect(call.input).toEqual({
+      type: "image",
+      data: imageData,
+      mimeType: "image/png",
+    });
+    expect(call.extraPayload).toEqual({
+      created_at: now,
+      kind: "image",
+      subject: "My Screenshot",
+    });
+  });
+
+  test("embeds indexed audio asset with correct modality", async () => {
+    const db = getDb();
+    const now = Date.now();
+    const filePath = join(testDir, "test-audio.mp3");
+    const audioData = Buffer.from("fake mp3 data");
+    writeFileSync(filePath, audioData);
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-audio",
+        title: "Podcast Episode",
+        filePath,
+        mimeType: "audio/mp3",
+        durationSeconds: 120,
+        fileHash: "hash-audio",
+        status: "indexed",
+        mediaType: "audio",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(makeJob({ assetId: "asset-audio" }), TEST_CONFIG);
+
+    expect(embedAndUpsertCalls).toHaveLength(1);
+    const call = embedAndUpsertCalls[0];
+    expect(call.targetType).toBe("media");
+    expect(call.targetId).toBe("asset-audio");
+    expect((call.input as { type: string }).type).toBe("audio");
+    expect((call.input as { mimeType: string }).mimeType).toBe("audio/mp3");
+  });
+
+  test("embeds indexed video asset with correct modality", async () => {
+    const db = getDb();
+    const now = Date.now();
+    const filePath = join(testDir, "test-video.mp4");
+    const videoData = Buffer.from("fake mp4 data");
+    writeFileSync(filePath, videoData);
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-video",
+        title: "Tutorial Video",
+        filePath,
+        mimeType: "video/mp4",
+        durationSeconds: 300,
+        fileHash: "hash-video",
+        status: "indexed",
+        mediaType: "video",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(makeJob({ assetId: "asset-video" }), TEST_CONFIG);
+
+    expect(embedAndUpsertCalls).toHaveLength(1);
+    const call = embedAndUpsertCalls[0];
+    expect(call.targetType).toBe("media");
+    expect(call.targetId).toBe("asset-video");
+    expect((call.input as { type: string }).type).toBe("video");
+    expect((call.input as { mimeType: string }).mimeType).toBe("video/mp4");
+  });
+});

package/src/memory/job-handlers/embedding.ts

@@ -1,10 +1,21 @@
+import { readFile } from "node:fs/promises";
+
 import { eq } from "drizzle-orm";
 
 import type { AssistantConfig } from "../../config/types.js";
+import { getConversationMemoryScopeId } from "../conversation-crud.js";
 import { getDb } from "../db.js";
+import type { EmbeddingInput } from "../embedding-types.js";
 import { asString, embedAndUpsert } from "../job-utils.js";
 import type { MemoryJob } from "../jobs-store.js";
-import { memoryItems, memorySegments, memorySummaries } from "../schema.js";
+import { extractMediaBlocks } from "../message-content.js";
+import {
+  mediaAssets,
+  memoryItems,
+  memorySegments,
+  memorySummaries,
+  messages,
+} from "../schema.js";
 
 export async function embedSegmentJob(
   job: MemoryJob,
@@ -75,3 +86,72 @@ export async function embedSummaryJob(
     },
   );
 }
+
+export async function embedMediaJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const assetId = asString(job.payload.assetId);
+  if (!assetId) return;
+
+  const db = getDb();
+  const asset = db
+    .select()
+    .from(mediaAssets)
+    .where(eq(mediaAssets.id, assetId))
+    .get();
+  if (!asset || asset.status !== "indexed") return;
+
+  // Read the media file from disk
+  const fileData = await readFile(asset.filePath);
+
+  // Determine modality from mediaType
+  const input: EmbeddingInput = {
+    type: asset.mediaType as "image" | "audio" | "video",
+    data: fileData,
+    mimeType: asset.mimeType,
+  };
+
+  await embedAndUpsert(config, "media", asset.id, input, {
+    created_at: asset.createdAt,
+    kind: asset.mediaType,
+    subject: asset.title,
+  });
+}
+
+export async function embedAttachmentJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const messageId = asString(job.payload.messageId);
+  const blockIndex = job.payload.blockIndex as number;
+  if (!messageId || typeof blockIndex !== "number") return;
+
+  const db = getDb();
+  const message = db
+    .select()
+    .from(messages)
+    .where(eq(messages.id, messageId))
+    .get();
+  if (!message) return;
+
+  const mediaBlocks = extractMediaBlocks(message.content);
+  const block = mediaBlocks.find((b) => b.index === blockIndex);
+  if (!block) return;
+
+  const input: EmbeddingInput = {
+    type: block.type,
+    data: block.data,
+    mimeType: block.mimeType,
+  };
+
+  // Use messageId + blockIndex as targetId for uniqueness
+  const targetId = `${messageId}:${blockIndex}`;
+  const memoryScopeId = getConversationMemoryScopeId(message.conversationId);
+  await embedAndUpsert(config, "media", targetId, input, {
+    created_at: message.createdAt,
+    message_id: messageId,
+    conversation_id: message.conversationId,
+    memory_scope_id: memoryScopeId,
+  });
+}

package/src/memory/job-handlers/index-maintenance.ts

@@ -1,16 +1,21 @@
-import { eq } from "drizzle-orm";
+import { eq, like } from "drizzle-orm";
 
+import { getConfig } from "../../config/loader.js";
 import { getLogger } from "../../util/logger.js";
 import { getDb, rawExec } from "../db.js";
+import { selectedBackendSupportsMultimodal } from "../embedding-backend.js";
 import { asString, BackendUnavailableError } from "../job-utils.js";
 import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
+import { extractMediaBlockMeta } from "../message-content.js";
 import { withQdrantBreaker } from "../qdrant-circuit-breaker.js";
 import { getQdrantClient } from "../qdrant-client.js";
 import {
+  mediaAssets,
   memoryEmbeddings,
   memoryItems,
   memorySegments,
   memorySummaries,
+  messages,
 } from "../schema.js";
 
 const log = getLogger("memory-jobs-worker");
@@ -48,6 +53,35 @@ export function rebuildIndexJob(): void {
   for (const segment of segments) {
     enqueueMemoryJob("embed_segment", { segmentId: segment.id });
   }
+
+  // Re-enqueue multimodal embedding jobs only when the resolved embedding
+  // backend supports multimodal inputs. Without this gate, embed_media and
+  // embed_attachment jobs would all fail for text-only backends.
+  if (selectedBackendSupportsMultimodal(getConfig())) {
+    const assets = db
+      .select({ id: mediaAssets.id })
+      .from(mediaAssets)
+      .where(eq(mediaAssets.status, "indexed"))
+      .all();
+    for (const asset of assets) {
+      enqueueMemoryJob("embed_media", { assetId: asset.id });
+    }
+
+    const imageMessages = db
+      .select({ id: messages.id, content: messages.content })
+      .from(messages)
+      .where(like(messages.content, '%"type":"image"%'))
+      .all();
+    for (const msg of imageMessages) {
+      const blocks = extractMediaBlockMeta(msg.content);
+      for (const block of blocks) {
+        enqueueMemoryJob("embed_attachment", {
+          messageId: msg.id,
+          blockIndex: block.index,
+        });
+      }
+    }
+  }
 }
 
 export async function deleteQdrantVectorsJob(job: MemoryJob): Promise<void> {

package/src/memory/job-handlers/media-processing.ts

@@ -6,9 +6,11 @@ import {
 import { mapSegmentsForAsset } from "../../config/bundled-skills/media-processing/tools/analyze-keyframes.js";
 import { preprocessForAsset } from "../../config/bundled-skills/media-processing/tools/extract-keyframes.js";
 import { reduceForAsset } from "../../config/bundled-skills/media-processing/tools/query-media-events.js";
+import { getConfig } from "../../config/loader.js";
 import { getLogger } from "../../util/logger.js";
+import { selectedBackendSupportsMultimodal } from "../embedding-backend.js";
 import { asString } from "../job-utils.js";
-import type { MemoryJob } from "../jobs-store.js";
+import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
 import { getMediaAssetById, updateMediaAssetStatus } from "../media-store.js";
 
 const log = getLogger("media-processing-job");
@@ -32,6 +34,9 @@ export async function mediaProcessingJob(job: MemoryJob): Promise<void> {
       "Skipping media processing pipeline — only video assets are supported",
     );
     updateMediaAssetStatus(mediaAssetId, "indexed");
+    if (selectedBackendSupportsMultimodal(getConfig())) {
+      enqueueMemoryJob("embed_media", { assetId: mediaAssetId });
+    }
     return;
   }
 
@@ -107,4 +112,9 @@ export async function mediaProcessingJob(job: MemoryJob): Promise<void> {
   if (result.cancelled) {
     throw new Error(`Media processing cancelled for asset ${mediaAssetId}`);
   }
+
+  updateMediaAssetStatus(mediaAssetId, "indexed");
+  if (selectedBackendSupportsMultimodal(getConfig())) {
+    enqueueMemoryJob("embed_media", { assetId: mediaAssetId });
+  }
 }

package/src/memory/job-utils.ts

@@ -1,4 +1,4 @@
-import { createHash, randomUUID } from "node:crypto";
+import { randomUUID } from "node:crypto";
 
 import { and, eq } from "drizzle-orm";
 
@@ -10,6 +10,11 @@ import {
   embedWithBackend,
   getMemoryBackendStatus,
 } from "./embedding-backend.js";
+import type { EmbeddingInput } from "./embedding-types.js";
+import {
+  embeddingInputContentHash,
+  normalizeEmbeddingInput,
+} from "./embedding-types.js";
 import { withQdrantBreaker } from "./qdrant-circuit-breaker.js";
 import { getQdrantClient } from "./qdrant-client.js";
 import { memoryEmbeddings } from "./schema.js";
@@ -136,9 +141,9 @@ export function truncate(text: string, max: number): string {
 
 export async function embedAndUpsert(
   config: AssistantConfig,
-  targetType: "segment" | "item" | "summary",
+  targetType: "segment" | "item" | "summary" | "media",
   targetId: string,
-  text: string,
+  input: EmbeddingInput,
   extraPayload?: Record<string, unknown>,
 ): Promise<void> {
   const status = getMemoryBackendStatus(config);
@@ -148,7 +153,7 @@ export async function embedAndUpsert(
     );
   }
 
-  const contentHash = createHash("sha256").update(text).digest("hex");
+  const contentHash = embeddingInputContentHash(input);
   let provider = status.provider;
   let model = status.model!;
   let vector: number[];
@@ -181,13 +186,21 @@ export async function embedAndUpsert(
       vector = JSON.parse(cachedRow.vectorJson!);
     }
   } else {
-    const embedded = await embedWithBackend(config, [text]);
+    const embedded = await embedWithBackend(config, [input]);
     vector = embedded.vectors[0];
     if (!vector) return;
     provider = embedded.provider;
     model = embedded.model;
   }
 
+  // Extract text for Qdrant payload: use the raw text for text inputs,
+  // or a description string for non-text (image/audio/video) inputs.
+  const normalized = normalizeEmbeddingInput(input);
+  const payloadText =
+    normalized.type === "text"
+      ? normalized.text
+      : `[${normalized.type}:${normalized.mimeType}]`;
+
   // Persist embedding in SQLite for cross-restart cache
   const now = Date.now();
   try {
@@ -234,9 +247,11 @@ export async function embedAndUpsert(
   }
 
   try {
+    const modality = normalized.type;
     await withQdrantBreaker(() =>
       qdrant.upsert(targetType, targetId, vector, {
-        text,
+        text: payloadText,
+        modality,
         created_at: (extraPayload?.created_at as number) ?? now,
         ...(extraPayload as Record<string, unknown> | undefined),
       }),

package/src/memory/jobs-store.ts

@@ -27,12 +27,16 @@ export type MemoryJobType =
   | "rebuild_index"
   | "reconcile_fts"
   | "delete_qdrant_vectors"
-  | "media_processing";
+  | "media_processing"
+  | "embed_media"
+  | "embed_attachment";
 
 const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_segment",
   "embed_item",
   "embed_summary",
+  "embed_media",
+  "embed_attachment",
 ];
 
 export interface MemoryJob<T = Record<string, unknown>> {

package/src/memory/jobs-worker.ts

@@ -18,7 +18,9 @@ import {
 } from "./job-handlers/conflict.js";
 // ── Per-job-type handlers ──────────────────────────────────────────
 import {
+  embedAttachmentJob,
   embedItemJob,
+  embedMediaJob,
   embedSegmentJob,
   embedSummaryJob,
 } from "./job-handlers/embedding.js";
@@ -334,6 +336,12 @@ async function processJob(
     case "media_processing":
       await mediaProcessingJob(job);
       return;
+    case "embed_media":
+      await embedMediaJob(job, config);
+      return;
+    case "embed_attachment":
+      await embedAttachmentJob(job, config);
+      return;
     default:
       throw new Error(
         `Unknown memory job type: ${(job as { type: string }).type}`,