@vellumai/assistant 0.4.44 → 0.4.45

package/src/memory/embedding-gemini.test.ts

@@ -0,0 +1,256 @@
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { GeminiEmbeddingBackend } from "./embedding-gemini.js";
+
+function makeSuccessResponse(values: number[]) {
+  return new Response(JSON.stringify({ embedding: { values } }), {
+    status: 200,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+describe("GeminiEmbeddingBackend", () => {
+  const originalFetch = globalThis.fetch;
+  let mockFetch: ReturnType<typeof mock>;
+
+  beforeEach(() => {
+    mockFetch = mock(() => Promise.resolve(makeSuccessResponse([0.1, 0.2, 0.3])));
+    globalThis.fetch = mockFetch as unknown as typeof fetch;
+  });
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  describe("text inputs", () => {
+    test("sends text as parts: [{ text }]", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed(["hello world"]);
+
+      expect(mockFetch).toHaveBeenCalledTimes(1);
+      const [url, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      expect(url).toContain("models/test-model:embedContent");
+      expect(url).toContain("key=test-key");
+
+      const body = JSON.parse(init.body as string);
+      expect(body.model).toBe("models/test-model");
+      expect(body.content).toEqual({ parts: [{ text: "hello world" }] });
+    });
+
+    test("handles TextEmbeddingInput objects", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed([{ type: "text", text: "structured text" }]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.content).toEqual({ parts: [{ text: "structured text" }] });
+    });
+  });
+
+  describe("image inputs", () => {
+    test("sends image as inline_data with base64", async () => {
+      const imageData = Buffer.from("fake-png-data");
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed([
+        { type: "image", data: imageData, mimeType: "image/png" },
+      ]);
+
+      expect(mockFetch).toHaveBeenCalledTimes(1);
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.content).toEqual({
+        parts: [
+          {
+            inline_data: {
+              mime_type: "image/png",
+              data: imageData.toString("base64"),
+            },
+          },
+        ],
+      });
+    });
+  });
+
+  describe("audio inputs", () => {
+    test("sends audio as inline_data with base64", async () => {
+      const audioData = Buffer.from("fake-audio-data");
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed([
+        { type: "audio", data: audioData, mimeType: "audio/mp3" },
+      ]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.content).toEqual({
+        parts: [
+          {
+            inline_data: {
+              mime_type: "audio/mp3",
+              data: audioData.toString("base64"),
+            },
+          },
+        ],
+      });
+    });
+  });
+
+  describe("video inputs", () => {
+    test("sends video as inline_data with base64", async () => {
+      const videoData = Buffer.from("fake-video-data");
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed([
+        { type: "video", data: videoData, mimeType: "video/mp4" },
+      ]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.content).toEqual({
+        parts: [
+          {
+            inline_data: {
+              mime_type: "video/mp4",
+              data: videoData.toString("base64"),
+            },
+          },
+        ],
+      });
+    });
+  });
+
+  describe("taskType and outputDimensionality", () => {
+    test("includes taskType in request body when configured", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model", {
+        taskType: "RETRIEVAL_DOCUMENT",
+      });
+      await backend.embed(["hello"]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.taskType).toBe("RETRIEVAL_DOCUMENT");
+    });
+
+    test("includes outputDimensionality in request body when dimensions configured", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model", {
+        dimensions: 256,
+      });
+      await backend.embed(["hello"]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.outputDimensionality).toBe(256);
+    });
+
+    test("includes both taskType and outputDimensionality when both configured", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model", {
+        taskType: "SEMANTIC_SIMILARITY",
+        dimensions: 512,
+      });
+      await backend.embed(["hello"]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.taskType).toBe("SEMANTIC_SIMILARITY");
+      expect(body.outputDimensionality).toBe(512);
+    });
+
+    test("omits taskType when not configured", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed(["hello"]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.taskType).toBeUndefined();
+    });
+
+    test("omits outputDimensionality when not configured", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed(["hello"]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.outputDimensionality).toBeUndefined();
+    });
+
+    test("omits both when options is undefined", async () => {
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await backend.embed(["hello"]);
+
+      const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
+      const body = JSON.parse(init.body as string);
+      expect(body.taskType).toBeUndefined();
+      expect(body.outputDimensionality).toBeUndefined();
+      expect(Object.keys(body)).toEqual(["model", "content"]);
+    });
+  });
+
+  describe("error handling", () => {
+    test("throws on non-OK response", async () => {
+      mockFetch = mock(() =>
+        Promise.resolve(
+          new Response("Internal Server Error", { status: 500 }),
+        ),
+      );
+      globalThis.fetch = mockFetch as unknown as typeof fetch;
+
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await expect(backend.embed(["hello"])).rejects.toThrow(
+        "Gemini embeddings request failed (500): Internal Server Error",
+      );
+    });
+
+    test("throws when response is missing embedding values", async () => {
+      mockFetch = mock(() =>
+        Promise.resolve(
+          new Response(JSON.stringify({}), {
+            status: 200,
+            headers: { "Content-Type": "application/json" },
+          }),
+        ),
+      );
+      globalThis.fetch = mockFetch as unknown as typeof fetch;
+
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await expect(backend.embed(["hello"])).rejects.toThrow(
+        "Gemini embeddings response missing vector values",
+      );
+    });
+
+    test("throws when embedding values array is empty", async () => {
+      mockFetch = mock(() =>
+        Promise.resolve(
+          new Response(JSON.stringify({ embedding: { values: [] } }), {
+            status: 200,
+            headers: { "Content-Type": "application/json" },
+          }),
+        ),
+      );
+      globalThis.fetch = mockFetch as unknown as typeof fetch;
+
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      await expect(backend.embed(["hello"])).rejects.toThrow(
+        "Gemini embeddings response missing vector values",
+      );
+    });
+  });
+
+  describe("multiple inputs", () => {
+    test("embeds multiple inputs sequentially", async () => {
+      let callCount = 0;
+      mockFetch = mock(() => {
+        callCount++;
+        return Promise.resolve(
+          makeSuccessResponse([0.1 * callCount, 0.2 * callCount]),
+        );
+      });
+      globalThis.fetch = mockFetch as unknown as typeof fetch;
+
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      const result = await backend.embed(["hello", "world"]);
+
+      expect(mockFetch).toHaveBeenCalledTimes(2);
+      expect(result).toHaveLength(2);
+      expect(result[0]).toEqual([0.1, 0.2]);
+      expect(result[1]).toEqual([0.2, 0.4]);
+    });
+  });
+});

package/src/memory/embedding-gemini.ts

@@ -1,7 +1,13 @@
 import type {
   EmbeddingBackend,
+  EmbeddingInput,
   EmbeddingRequestOptions,
 } from "./embedding-backend.js";
+import type {
+  EmbeddingTaskType,
+  MultimodalEmbeddingInput,
+} from "./embedding-types.js";
+import { normalizeEmbeddingInput } from "./embedding-types.js";
 
 interface GeminiEmbedResponse {
   embedding?: {
@@ -13,46 +19,59 @@ export class GeminiEmbeddingBackend implements EmbeddingBackend {
   readonly provider = "gemini" as const;
   readonly model: string;
   private readonly apiKey: string;
+  private readonly taskType?: EmbeddingTaskType;
+  private readonly dimensions?: number;
 
-  constructor(apiKey: string, model: string) {
+  constructor(
+    apiKey: string,
+    model: string,
+    options?: { taskType?: EmbeddingTaskType; dimensions?: number },
+  ) {
     this.apiKey = apiKey;
     this.model = model;
+    this.taskType = options?.taskType;
+    this.dimensions = options?.dimensions;
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
     const vectors: number[][] = [];
-    for (const text of texts) {
-      const values = await this.embedSingle(text, options);
+    for (const input of inputs) {
+      const values = await this.embedSingle(input, options);
       vectors.push(values);
     }
     return vectors;
   }
 
   private async embedSingle(
-    text: string,
+    input: EmbeddingInput,
     options?: EmbeddingRequestOptions,
   ): Promise<number[]> {
+    const normalized = normalizeEmbeddingInput(input);
+    const parts = this.buildParts(normalized);
+
+    const body: Record<string, unknown> = {
+      model: `models/${this.model}`,
+      content: { parts },
+    };
+    if (this.taskType) body.taskType = this.taskType;
+    if (this.dimensions) body.outputDimensionality = this.dimensions;
+
     const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(
       this.model,
     )}:embedContent?key=${encodeURIComponent(this.apiKey)}`;
     const response = await fetch(url, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({
-        model: `models/${this.model}`,
-        content: {
-          parts: [{ text }],
-        },
-      }),
+      body: JSON.stringify(body),
       signal: options?.signal,
     });
     if (!response.ok) {
-      const body = await response.text();
+      const responseBody = await response.text();
       throw new Error(
-        `Gemini embeddings request failed (${response.status}): ${body}`,
+        `Gemini embeddings request failed (${response.status}): ${responseBody}`,
       );
     }
     const payload = (await response.json()) as GeminiEmbedResponse;
@@ -62,4 +81,19 @@ export class GeminiEmbeddingBackend implements EmbeddingBackend {
     }
     return values;
   }
+
+  private buildParts(input: MultimodalEmbeddingInput): unknown[] {
+    if (input.type === "text") {
+      return [{ text: input.text }];
+    }
+    // Image, audio, video: use inline_data with base64
+    return [
+      {
+        inline_data: {
+          mime_type: input.mimeType,
+          data: input.data.toString("base64"),
+        },
+      },
+    ];
+  }
 }

package/src/memory/embedding-local.ts

@@ -6,9 +6,11 @@ import { getEmbeddingModelsDir, getRootDir } from "../util/platform.js";
 import { PromiseGuard } from "../util/promise-guard.js";
 import type {
   EmbeddingBackend,
+  EmbeddingInput,
   EmbeddingRequestOptions,
 } from "./embedding-backend.js";
 import { EmbeddingRuntimeManager } from "./embedding-runtime-manager.js";
+import { normalizeEmbeddingInput } from "./embedding-types.js";
 
 const log = getLogger("memory-embedding-local");
 
@@ -56,10 +58,20 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
-    if (texts.length === 0) return [];
+    if (inputs.length === 0) return [];
+
+    const texts = inputs.map((i) => {
+      const n = normalizeEmbeddingInput(i);
+      if (n.type !== "text") {
+        throw new Error(
+          "Local embedding backend only supports text inputs",
+        );
+      }
+      return n.text;
+    });
     if (options?.signal?.aborted)
       throw new DOMException("Aborted", "AbortError");
 

package/src/memory/embedding-ollama.ts

@@ -1,8 +1,10 @@
 import { getOllamaBaseUrlEnv } from "../config/env.js";
 import type {
   EmbeddingBackend,
+  EmbeddingInput,
   EmbeddingRequestOptions,
 } from "./embedding-backend.js";
+import { normalizeEmbeddingInput } from "./embedding-types.js";
 
 interface OllamaEmbeddingsResponse {
   data?: Array<{ embedding: number[] }>;
@@ -24,10 +26,21 @@ export class OllamaEmbeddingBackend implements EmbeddingBackend {
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
-    if (texts.length === 0) return [];
+    if (inputs.length === 0) return [];
+
+    const texts = inputs.map((i) => {
+      const n = normalizeEmbeddingInput(i);
+      if (n.type !== "text") {
+        throw new Error(
+          "Ollama embedding backend only supports text inputs",
+        );
+      }
+      return n.text;
+    });
+
     const response = await fetch(`${this.baseUrl}/embeddings`, {
       method: "POST",
       headers: {

package/src/memory/embedding-openai.ts

@@ -2,8 +2,10 @@ import OpenAI from "openai";
 
 import type {
   EmbeddingBackend,
+  EmbeddingInput,
   EmbeddingRequestOptions,
 } from "./embedding-backend.js";
+import { normalizeEmbeddingInput } from "./embedding-types.js";
 
 export class OpenAIEmbeddingBackend implements EmbeddingBackend {
   readonly provider = "openai" as const;
@@ -16,10 +18,21 @@ export class OpenAIEmbeddingBackend implements EmbeddingBackend {
   }
 
   async embed(
-    texts: string[],
+    inputs: EmbeddingInput[],
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
-    if (texts.length === 0) return [];
+    if (inputs.length === 0) return [];
+
+    const texts = inputs.map((i) => {
+      const n = normalizeEmbeddingInput(i);
+      if (n.type !== "text") {
+        throw new Error(
+          "OpenAI embedding backend only supports text inputs",
+        );
+      }
+      return n.text;
+    });
+
     const response = await this.client.embeddings.create(
       {
         model: this.model,

package/src/memory/embedding-types.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  embeddingInputContentHash,
+  type MultimodalEmbeddingInput,
+  normalizeEmbeddingInput,
+} from "./embedding-types.js";
+
+describe("normalizeEmbeddingInput", () => {
+  test("converts a raw string to a TextEmbeddingInput", () => {
+    const result = normalizeEmbeddingInput("hello");
+    expect(result).toEqual({ type: "text", text: "hello" });
+  });
+
+  test("passes through a TextEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = { type: "text", text: "hello" };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toEqual({ type: "text", text: "hello" });
+  });
+
+  test("passes through an ImageEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = {
+      type: "image",
+      data: Buffer.from("fake-png"),
+      mimeType: "image/png",
+    };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toBe(input);
+  });
+
+  test("passes through an AudioEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = {
+      type: "audio",
+      data: Buffer.from("fake-audio"),
+      mimeType: "audio/mp3",
+    };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toBe(input);
+  });
+
+  test("passes through a VideoEmbeddingInput unchanged", () => {
+    const input: MultimodalEmbeddingInput = {
+      type: "video",
+      data: Buffer.from("fake-video"),
+      mimeType: "video/mp4",
+    };
+    const result = normalizeEmbeddingInput(input);
+    expect(result).toBe(input);
+  });
+});
+
+describe("embeddingInputContentHash", () => {
+  test("produces consistent hash for the same text input", () => {
+    const hash1 = embeddingInputContentHash("hello");
+    const hash2 = embeddingInputContentHash("hello");
+    expect(hash1).toBe(hash2);
+  });
+
+  test("produces same hash for raw string and equivalent TextEmbeddingInput", () => {
+    const hash1 = embeddingInputContentHash("hello");
+    const hash2 = embeddingInputContentHash({ type: "text", text: "hello" });
+    expect(hash1).toBe(hash2);
+  });
+
+  test("produces different hashes for different text inputs", () => {
+    const hash1 = embeddingInputContentHash("hello");
+    const hash2 = embeddingInputContentHash("world");
+    expect(hash1).not.toBe(hash2);
+  });
+
+  test("produces distinct hashes for text vs binary inputs with same byte content", () => {
+    const content = "hello";
+    const textHash = embeddingInputContentHash(content);
+    const imageHash = embeddingInputContentHash({
+      type: "image",
+      data: Buffer.from(content),
+      mimeType: "image/png",
+    });
+    expect(textHash).not.toBe(imageHash);
+  });
+
+  test("produces different hashes for same data but different mime types", () => {
+    const data = Buffer.from("same-data");
+    const hash1 = embeddingInputContentHash({
+      type: "image",
+      data,
+      mimeType: "image/png",
+    });
+    const hash2 = embeddingInputContentHash({
+      type: "image",
+      data,
+      mimeType: "image/jpeg",
+    });
+    expect(hash1).not.toBe(hash2);
+  });
+
+  test("produces different hashes for same data but different modality types", () => {
+    const data = Buffer.from("same-data");
+    const imageHash = embeddingInputContentHash({
+      type: "image",
+      data,
+      mimeType: "application/octet-stream",
+    });
+    const audioHash = embeddingInputContentHash({
+      type: "audio",
+      data,
+      mimeType: "application/octet-stream",
+    });
+    expect(imageHash).not.toBe(audioHash);
+  });
+
+  test("returns a hex string", () => {
+    const hash = embeddingInputContentHash("test");
+    expect(hash).toMatch(/^[0-9a-f]{64}$/);
+  });
+});

package/src/memory/embedding-types.ts

@@ -0,0 +1,61 @@
+import { createHash } from "crypto";
+
+export type EmbeddingTaskType =
+  | "SEMANTIC_SIMILARITY"
+  | "CLASSIFICATION"
+  | "CLUSTERING"
+  | "RETRIEVAL_DOCUMENT"
+  | "RETRIEVAL_QUERY"
+  | "CODE_RETRIEVAL_QUERY"
+  | "QUESTION_ANSWERING"
+  | "FACT_VERIFICATION";
+
+export interface TextEmbeddingInput {
+  type: "text";
+  text: string;
+}
+
+export interface ImageEmbeddingInput {
+  type: "image";
+  data: Buffer;
+  mimeType: string; // "image/png" | "image/jpeg"
+}
+
+export interface AudioEmbeddingInput {
+  type: "audio";
+  data: Buffer;
+  mimeType: string; // "audio/mp3" | "audio/wav"
+}
+
+export interface VideoEmbeddingInput {
+  type: "video";
+  data: Buffer;
+  mimeType: string; // "video/mp4" | "video/mov"
+}
+
+export type MultimodalEmbeddingInput =
+  | TextEmbeddingInput
+  | ImageEmbeddingInput
+  | AudioEmbeddingInput
+  | VideoEmbeddingInput;
+
+/** Accepts raw strings as shorthand for text inputs. */
+export type EmbeddingInput = string | MultimodalEmbeddingInput;
+
+export function normalizeEmbeddingInput(input: EmbeddingInput): MultimodalEmbeddingInput {
+  if (typeof input === "string") return { type: "text", text: input };
+  return input;
+}
+
+export function embeddingInputContentHash(input: EmbeddingInput): string {
+  const normalized = normalizeEmbeddingInput(input);
+  const hash = createHash("sha256");
+  hash.update(normalized.type);
+  if (normalized.type === "text") {
+    hash.update(normalized.text);
+  } else {
+    hash.update(normalized.mimeType);
+    hash.update(normalized.data);
+  }
+  return hash.digest("hex");
+}

package/src/memory/fingerprint.ts

@@ -5,7 +5,7 @@ import { createHash } from "node:crypto";
  *
  * Format: sha256(`${scopeId}|${kind}|${subject}|${statement}`)
  *
- * All writers (memory_save, memory_update, items-extractor, playbook-create,
+ * All writers (memory_manage save/update ops, items-extractor, playbook-create,
  * playbook-update, gmail-analyze-style) MUST use this function so the
  * fingerprint scheme stays consistent and deduplication works correctly.
  */