@vellumai/assistant 0.4.43 → 0.4.45

package/src/memory/embedding-ollama.ts

@@ -1,8 +1,10 @@
 import { getOllamaBaseUrlEnv } from "../config/env.js";
 import type {
   EmbeddingBackend,
+  EmbeddingInput,
   EmbeddingRequestOptions,
 } from "./embedding-backend.js";
+import { normalizeEmbeddingInput } from "./embedding-types.js";
 
 interface OllamaEmbeddingsResponse {
   data?: Array<{ embedding: number[] }>;
@@ -24,10 +26,21 @@ export class OllamaEmbeddingBackend implements EmbeddingBackend {
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
-    if (texts.length === 0) return [];
+    if (inputs.length === 0) return [];
+
+    const texts = inputs.map((i) => {
+      const n = normalizeEmbeddingInput(i);
+      if (n.type !== "text") {
+        throw new Error(
+          "Ollama embedding backend only supports text inputs",
+        );
+      }
+      return n.text;
+    });
+
     const response = await fetch(`${this.baseUrl}/embeddings`, {
       method: "POST",
       headers: {

package/src/memory/embedding-openai.ts

@@ -2,8 +2,10 @@ import OpenAI from "openai";
 
 import type {
   EmbeddingBackend,
+  EmbeddingInput,
   EmbeddingRequestOptions,
 } from "./embedding-backend.js";
+import { normalizeEmbeddingInput } from "./embedding-types.js";
 
 export class OpenAIEmbeddingBackend implements EmbeddingBackend {
   readonly provider = "openai" as const;
@@ -16,10 +18,21 @@ export class OpenAIEmbeddingBackend implements EmbeddingBackend {
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
-    if (texts.length === 0) return [];
+    if (inputs.length === 0) return [];
+
+    const texts = inputs.map((i) => {
+      const n = normalizeEmbeddingInput(i);
+      if (n.type !== "text") {
+        throw new Error(
+          "OpenAI embedding backend only supports text inputs",
+        );
+      }
+      return n.text;
+    });
+
     const response = await this.client.embeddings.create(
       {
         model: this.model,

package/src/memory/embedding-types.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  embeddingInputContentHash,
+  type MultimodalEmbeddingInput,
+  normalizeEmbeddingInput,
+} from "./embedding-types.js";
+
+describe("normalizeEmbeddingInput", () => {
+  test("converts a raw string to a TextEmbeddingInput", () => {
+    const result = normalizeEmbeddingInput("hello");
+    expect(result).toEqual({ type: "text", text: "hello" });
+  });
+
+  test("passes through a TextEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = { type: "text", text: "hello" };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toEqual({ type: "text", text: "hello" });
+  });
+
+  test("passes through an ImageEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = {
+      type: "image",
+      data: Buffer.from("fake-png"),
+      mimeType: "image/png",
+    };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toBe(input);
+  });
+
+  test("passes through an AudioEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = {
+      type: "audio",
+      data: Buffer.from("fake-audio"),
+      mimeType: "audio/mp3",
+    };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toBe(input);
+  });
+
+  test("passes through a VideoEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = {
+      type: "video",
+      data: Buffer.from("fake-video"),
+      mimeType: "video/mp4",
+    };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toBe(input);
+  });
+});
+
+describe("embeddingInputContentHash", () => {
+  test("produces consistent hash for the same text input", () => {
+    const hash1 = embeddingInputContentHash("hello");
+    const hash2 = embeddingInputContentHash("hello");
+    expect(hash1).toBe(hash2);
+  });
+
+  test("produces same hash for raw string and equivalent TextEmbeddingInput", () => {
+    const hash1 = embeddingInputContentHash("hello");
+    const hash2 = embeddingInputContentHash({ type: "text", text: "hello" });
+    expect(hash1).toBe(hash2);
+  });
+
+  test("produces different hashes for different text inputs", () => {
+    const hash1 = embeddingInputContentHash("hello");
+    const hash2 = embeddingInputContentHash("world");
+    expect(hash1).not.toBe(hash2);
+  });
+
+  test("produces distinct hashes for text vs binary inputs with same byte content", () => {
+    const content = "hello";
+    const textHash = embeddingInputContentHash(content);
+    const imageHash = embeddingInputContentHash({
+      type: "image",
+      data: Buffer.from(content),
+      mimeType: "image/png",
+    });
+    expect(textHash).not.toBe(imageHash);
+  });
+
+  test("produces different hashes for same data but different mime types", () => {
+    const data = Buffer.from("same-data");
+    const hash1 = embeddingInputContentHash({
+      type: "image",
+      data,
+      mimeType: "image/png",
+    });
+    const hash2 = embeddingInputContentHash({
+      type: "image",
+      data,
+      mimeType: "image/jpeg",
+    });
+    expect(hash1).not.toBe(hash2);
+  });
+
+  test("produces different hashes for same data but different modality types", () => {
+    const data = Buffer.from("same-data");
+    const imageHash = embeddingInputContentHash({
+      type: "image",
+      data,
+      mimeType: "application/octet-stream",
+    });
+    const audioHash = embeddingInputContentHash({
+      type: "audio",
+      data,
+      mimeType: "application/octet-stream",
+    });
+    expect(imageHash).not.toBe(audioHash);
+  });
+
+  test("returns a hex string", () => {
+    const hash = embeddingInputContentHash("test");
+    expect(hash).toMatch(/^[0-9a-f]{64}$/);
+  });
+});

package/src/memory/embedding-types.ts

@@ -0,0 +1,61 @@
+import { createHash } from "crypto";
+
+export type EmbeddingTaskType =
+  | "SEMANTIC_SIMILARITY"
+  | "CLASSIFICATION"
+  | "CLUSTERING"
+  | "RETRIEVAL_DOCUMENT"
+  | "RETRIEVAL_QUERY"
+  | "CODE_RETRIEVAL_QUERY"
+  | "QUESTION_ANSWERING"
+  | "FACT_VERIFICATION";
+
+export interface TextEmbeddingInput {
+  type: "text";
+  text: string;
+}
+
+export interface ImageEmbeddingInput {
+  type: "image";
+  data: Buffer;
+  mimeType: string; // "image/png" | "image/jpeg"
+}
+
+export interface AudioEmbeddingInput {
+  type: "audio";
+  data: Buffer;
+  mimeType: string; // "audio/mp3" | "audio/wav"
+}
+
+export interface VideoEmbeddingInput {
+  type: "video";
+  data: Buffer;
+  mimeType: string; // "video/mp4" | "video/mov"
+}
+
+export type MultimodalEmbeddingInput =
+  | TextEmbeddingInput
+  | ImageEmbeddingInput
+  | AudioEmbeddingInput
+  | VideoEmbeddingInput;
+
+/** Accepts raw strings as shorthand for text inputs. */
+export type EmbeddingInput = string | MultimodalEmbeddingInput;
+
+export function normalizeEmbeddingInput(input: EmbeddingInput): MultimodalEmbeddingInput {
+  if (typeof input === "string") return { type: "text", text: input };
+  return input;
+}
+
+export function embeddingInputContentHash(input: EmbeddingInput): string {
+  const normalized = normalizeEmbeddingInput(input);
+  const hash = createHash("sha256");
+  hash.update(normalized.type);
+  if (normalized.type === "text") {
+    hash.update(normalized.text);
+  } else {
+    hash.update(normalized.mimeType);
+    hash.update(normalized.data);
+  }
+  return hash.digest("hex");
+}

package/src/memory/fingerprint.ts

@@ -5,7 +5,7 @@ import { createHash } from "node:crypto";
  *
  * Format: sha256(`${scopeId}|${kind}|${subject}|${statement}`)
  *
- * All writers (memory_save, memory_update, items-extractor, playbook-create,
+ * All writers (memory_manage save/update ops, items-extractor, playbook-create,
  * playbook-update, gmail-analyze-style) MUST use this function so the
  * fingerprint scheme stays consistent and deduplication works correctly.
  */

package/src/memory/indexer.ts

@@ -1,16 +1,21 @@
 import { createHash } from "crypto";
 import { desc, eq } from "drizzle-orm";
 
+import { getConfig } from "../config/loader.js";
 import type { MemoryConfig } from "../config/types.js";
 import type { TrustClass } from "../runtime/actor-trust-resolver.js";
 import { getLogger } from "../util/logger.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
 import { getDb } from "./db.js";
+import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
 import {
   enqueueMemoryJob,
   enqueueResolvePendingConflictsForMessageJob,
 } from "./jobs-store.js";
-import { extractTextFromStoredMessageContent } from "./message-content.js";
+import {
+  extractMediaBlocks,
+  extractTextFromStoredMessageContent,
+} from "./message-content.js";
 import { bumpMemoryVersion } from "./recall-cache.js";
 import { memorySegments } from "./schema.js";
 import { segmentText } from "./segmenter.js";
@@ -73,6 +78,13 @@ export function indexMessageNow(
   const shouldResolveConflicts =
     input.role === "user" && config.conflicts.enabled;
 
+  // Check if the resolved embedding backend supports multimodal input.
+  // Only enqueue embed_attachment jobs when it does (currently Gemini only).
+  const supportsMultimodal = selectedBackendSupportsMultimodal(getConfig());
+  const mediaBlocks = supportsMultimodal
+    ? extractMediaBlocks(input.content).filter((b) => b.type === "image")
+    : [];
+
   // Wrap all segment inserts and job enqueues in a single transaction so they
   // either all succeed or all roll back, preventing partial/orphaned state.
   let skippedEmbedJobs = 0;
@@ -121,6 +133,17 @@ export function indexMessageNow(
       }
     }
 
+    // Enqueue embed_attachment jobs for image content blocks when the
+    // embedding provider supports multimodal (Gemini only).
+    for (const block of mediaBlocks) {
+      enqueueMemoryJob(
+        "embed_attachment",
+        { messageId: input.messageId, blockIndex: block.index },
+        Date.now(),
+        tx,
+      );
+    }
+
     if (shouldExtract && isTrustedActor) {
       enqueueMemoryJob(
         "extract_items",
@@ -168,6 +191,7 @@ export function indexMessageNow(
   const enqueuedJobs =
     segments.length -
     skippedEmbedJobs +
+    mediaBlocks.length +
     (shouldExtract && !extractionGated ? 2 : 1) +
     (shouldResolveConflicts && !extractionGated ? 1 : 0);
   return {

package/src/memory/job-handlers/embedding.test.ts

@@ -0,0 +1,258 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  mock,
+  test,
+} from "bun:test";
+
+const testDir = mkdtempSync(join(tmpdir(), "embed-media-test-"));
+
+mock.module("../../util/platform.js", () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Track calls to embedAndUpsert
+const embedAndUpsertCalls: Array<{
+  config: unknown;
+  targetType: string;
+  targetId: string;
+  input: unknown;
+  extraPayload: unknown;
+}> = [];
+
+mock.module("../job-utils.js", () => ({
+  asString: (value: unknown) =>
+    typeof value === "string" && value.length > 0 ? value : null,
+  embedAndUpsert: async (
+    config: unknown,
+    targetType: string,
+    targetId: string,
+    input: unknown,
+    extraPayload: unknown,
+  ) => {
+    embedAndUpsertCalls.push({
+      config,
+      targetType,
+      targetId,
+      input,
+      extraPayload,
+    });
+  },
+}));
+
+import { DEFAULT_CONFIG } from "../../config/defaults.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { getDb, initializeDb, resetDb } from "../db.js";
+import type { MemoryJob } from "../jobs-store.js";
+import { mediaAssets } from "../schema.js";
+import { embedMediaJob } from "./embedding.js";
+
+const TEST_CONFIG: AssistantConfig = {
+  ...DEFAULT_CONFIG,
+  memory: {
+    ...DEFAULT_CONFIG.memory,
+    extraction: {
+      ...DEFAULT_CONFIG.memory.extraction,
+      useLLM: false,
+    },
+  },
+};
+
+describe("embedMediaJob", () => {
+  beforeAll(() => {
+    initializeDb();
+  });
+
+  beforeEach(() => {
+    embedAndUpsertCalls.length = 0;
+    resetDb();
+    initializeDb();
+  });
+
+  afterAll(() => {
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  function makeJob(payload: Record<string, unknown>): MemoryJob {
+    return {
+      id: "job-1",
+      type: "embed_media",
+      payload,
+      status: "running",
+      attempts: 0,
+      deferrals: 0,
+      runAfter: 0,
+      lastError: null,
+      startedAt: Date.now(),
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+    };
+  }
+
+  test("skips when assetId is missing", async () => {
+    await embedMediaJob(makeJob({}), TEST_CONFIG);
+    expect(embedAndUpsertCalls).toHaveLength(0);
+  });
+
+  test("skips when asset is not found", async () => {
+    await embedMediaJob(makeJob({ assetId: "nonexistent" }), TEST_CONFIG);
+    expect(embedAndUpsertCalls).toHaveLength(0);
+  });
+
+  test("skips when asset status is not indexed", async () => {
+    const db = getDb();
+    const now = Date.now();
+    // Create a temp file
+    const filePath = join(testDir, "test-registered.png");
+    writeFileSync(filePath, Buffer.from("fake image data"));
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-registered",
+        title: "Test Image",
+        filePath,
+        mimeType: "image/png",
+        durationSeconds: null,
+        fileHash: "hash-registered",
+        status: "registered",
+        mediaType: "image",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(
+      makeJob({ assetId: "asset-registered" }),
+      TEST_CONFIG,
+    );
+    expect(embedAndUpsertCalls).toHaveLength(0);
+  });
+
+  test("embeds indexed image asset with correct input type and target", async () => {
+    const db = getDb();
+    const now = Date.now();
+    const filePath = join(testDir, "test-image.png");
+    const imageData = Buffer.from("fake png data");
+    writeFileSync(filePath, imageData);
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-image",
+        title: "My Screenshot",
+        filePath,
+        mimeType: "image/png",
+        durationSeconds: null,
+        fileHash: "hash-image",
+        status: "indexed",
+        mediaType: "image",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(makeJob({ assetId: "asset-image" }), TEST_CONFIG);
+
+    expect(embedAndUpsertCalls).toHaveLength(1);
+    const call = embedAndUpsertCalls[0];
+    expect(call.targetType).toBe("media");
+    expect(call.targetId).toBe("asset-image");
+    expect(call.input).toEqual({
+      type: "image",
+      data: imageData,
+      mimeType: "image/png",
+    });
+    expect(call.extraPayload).toEqual({
+      created_at: now,
+      kind: "image",
+      subject: "My Screenshot",
+    });
+  });
+
+  test("embeds indexed audio asset with correct modality", async () => {
+    const db = getDb();
+    const now = Date.now();
+    const filePath = join(testDir, "test-audio.mp3");
+    const audioData = Buffer.from("fake mp3 data");
+    writeFileSync(filePath, audioData);
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-audio",
+        title: "Podcast Episode",
+        filePath,
+        mimeType: "audio/mp3",
+        durationSeconds: 120,
+        fileHash: "hash-audio",
+        status: "indexed",
+        mediaType: "audio",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(makeJob({ assetId: "asset-audio" }), TEST_CONFIG);
+
+    expect(embedAndUpsertCalls).toHaveLength(1);
+    const call = embedAndUpsertCalls[0];
+    expect(call.targetType).toBe("media");
+    expect(call.targetId).toBe("asset-audio");
+    expect((call.input as { type: string }).type).toBe("audio");
+    expect((call.input as { mimeType: string }).mimeType).toBe("audio/mp3");
+  });
+
+  test("embeds indexed video asset with correct modality", async () => {
+    const db = getDb();
+    const now = Date.now();
+    const filePath = join(testDir, "test-video.mp4");
+    const videoData = Buffer.from("fake mp4 data");
+    writeFileSync(filePath, videoData);
+
+    db.insert(mediaAssets)
+      .values({
+        id: "asset-video",
+        title: "Tutorial Video",
+        filePath,
+        mimeType: "video/mp4",
+        durationSeconds: 300,
+        fileHash: "hash-video",
+        status: "indexed",
+        mediaType: "video",
+        metadata: null,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    await embedMediaJob(makeJob({ assetId: "asset-video" }), TEST_CONFIG);
+
+    expect(embedAndUpsertCalls).toHaveLength(1);
+    const call = embedAndUpsertCalls[0];
+    expect(call.targetType).toBe("media");
+    expect(call.targetId).toBe("asset-video");
+    expect((call.input as { type: string }).type).toBe("video");
+    expect((call.input as { mimeType: string }).mimeType).toBe("video/mp4");
+  });
+});

package/src/memory/job-handlers/embedding.ts

@@ -1,10 +1,21 @@
+import { readFile } from "node:fs/promises";
+
 import { eq } from "drizzle-orm";
 
 import type { AssistantConfig } from "../../config/types.js";
+import { getConversationMemoryScopeId } from "../conversation-crud.js";
 import { getDb } from "../db.js";
+import type { EmbeddingInput } from "../embedding-types.js";
 import { asString, embedAndUpsert } from "../job-utils.js";
 import type { MemoryJob } from "../jobs-store.js";
-import { memoryItems, memorySegments, memorySummaries } from "../schema.js";
+import { extractMediaBlocks } from "../message-content.js";
+import {
+  mediaAssets,
+  memoryItems,
+  memorySegments,
+  memorySummaries,
+  messages,
+} from "../schema.js";
 
 export async function embedSegmentJob(
   job: MemoryJob,
@@ -75,3 +86,72 @@ export async function embedSummaryJob(
     },
   );
 }
+
+export async function embedMediaJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const assetId = asString(job.payload.assetId);
+  if (!assetId) return;
+
+  const db = getDb();
+  const asset = db
+    .select()
+    .from(mediaAssets)
+    .where(eq(mediaAssets.id, assetId))
+    .get();
+  if (!asset || asset.status !== "indexed") return;
+
+  // Read the media file from disk
+  const fileData = await readFile(asset.filePath);
+
+  // Determine modality from mediaType
+  const input: EmbeddingInput = {
+    type: asset.mediaType as "image" | "audio" | "video",
+    data: fileData,
+    mimeType: asset.mimeType,
+  };
+
+  await embedAndUpsert(config, "media", asset.id, input, {
+    created_at: asset.createdAt,
+    kind: asset.mediaType,
+    subject: asset.title,
+  });
+}
+
+export async function embedAttachmentJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const messageId = asString(job.payload.messageId);
+  const blockIndex = job.payload.blockIndex as number;
+  if (!messageId || typeof blockIndex !== "number") return;
+
+  const db = getDb();
+  const message = db
+    .select()
+    .from(messages)
+    .where(eq(messages.id, messageId))
+    .get();
+  if (!message) return;
+
+  const mediaBlocks = extractMediaBlocks(message.content);
+  const block = mediaBlocks.find((b) => b.index === blockIndex);
+  if (!block) return;
+
+  const input: EmbeddingInput = {
+    type: block.type,
+    data: block.data,
+    mimeType: block.mimeType,
+  };
+
+  // Use messageId + blockIndex as targetId for uniqueness
+  const targetId = `${messageId}:${blockIndex}`;
+  const memoryScopeId = getConversationMemoryScopeId(message.conversationId);
+  await embedAndUpsert(config, "media", targetId, input, {
+    created_at: message.createdAt,
+    message_id: messageId,
+    conversation_id: message.conversationId,
+    memory_scope_id: memoryScopeId,
+  });
+}