@convex-dev/agent 0.1.6 → 0.1.7-alpha.1

package/src/mapping.test.ts

@@ -0,0 +1,163 @@
+import { describe, test, expect } from "vitest";
+import {
+  guessMimeType,
+  serializeDataOrUrl,
+  deserializeUrl,
+  serializeMessage,
+  deserializeMessage,
+  serializeContent,
+} from "./mapping";
+import { api } from "./component/_generated/api.js";
+import type { AgentComponent, ActionCtx } from "./client/types";
+import { vMessage } from "./validators";
+import fs from "fs";
+import path from "path";
+import type { SerializedContent } from "./mapping";
+import { validate } from "convex-helpers/validators";
+
+const testAssetsDir = path.join(__dirname, "../test-assets");
+const testFiles = [
+  "book.svg",
+  "bump.jpeg",
+  "stack.png",
+  "favicon.ico",
+  "convex-logo.svg",
+  "stack-light@3x.webp",
+];
+
+function fileToArrayBuffer(filePath: string): ArrayBuffer {
+  const buf = fs.readFileSync(filePath);
+  return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+}
+
+describe("mapping", () => {
+  test("infers correct mimeType for all test-assets", () => {
+    const expected: { [key: string]: string } = {
+      "book.svg": "image/svg+xml", // <svg
+      "bump.jpeg": "image/jpeg",
+      "stack.png": "image/png",
+      "favicon.ico": "application/octet-stream", // fallback for ico
+      "convex-logo.svg": "image/svg+xml", // <?xm
+      "stack-light@3x.webp": "image/webp",
+      "cat.gif": "image/gif",
+    };
+    for (const file of testFiles) {
+      const ab = fileToArrayBuffer(path.join(testAssetsDir, file));
+      const mime = guessMimeType(ab);
+      expect(mime).toBe(expected[file]);
+    }
+  });
+
+  test("turns Uint8Array into ArrayBuffer and round-trips", () => {
+    const arr = new Uint8Array([1, 2, 3, 4, 5]);
+    // serializeDataOrUrl should return the same ArrayBuffer
+    const ser = serializeDataOrUrl(arr);
+    expect(ser).toBeInstanceOf(ArrayBuffer);
+    expect(new Uint8Array(ser as ArrayBuffer)).toEqual(arr);
+    // deserializeUrl should return the same ArrayBuffer
+    const deser = deserializeUrl(ser);
+    expect(deser).toBeInstanceOf(ArrayBuffer);
+    expect(new Uint8Array(deser as ArrayBuffer)).toEqual(arr);
+  });
+
+  test("round-trip serialize/deserialize message", async () => {
+    const message = {
+      role: "user" as const,
+      content: "hello world",
+      providerOptions: {},
+    };
+    // Fake ctx and component
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async () => undefined,
+      storage: {
+        store: async () => "storageId",
+        getUrl: async () => "https://example.com/file",
+        delete: async () => undefined,
+      },
+    } as unknown as ActionCtx;
+    const component = api as unknown as AgentComponent;
+    const { message: ser } = await serializeMessage(ctx, component, message);
+    // Use is for type validation
+    expect(validate(vMessage, ser)).toBeTruthy();
+    const round = deserializeMessage(ser);
+    expect(round).toEqual(message);
+  });
+
+  test("saving files returns fileIds when too big", async () => {
+    // Make a big file
+    const bigArr = new Uint8Array(1024 * 65).fill(1);
+    const ab = bigArr.buffer.slice(
+      bigArr.byteOffset,
+      bigArr.byteOffset + bigArr.byteLength
+    );
+    let called = false;
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async (_fn: unknown, _args: unknown) => {
+        called = true;
+        return { fileId: "file-123", storageId: "storage-123" };
+      },
+      storage: {
+        store: async () => "storageId",
+        getUrl: async () => "https://example.com/file",
+        delete: async () => undefined,
+      },
+    } as unknown as ActionCtx;
+    const component = api as unknown as AgentComponent;
+    const content = [
+      {
+        type: "file" as const,
+        data: ab,
+        filename: "bigfile.bin",
+        mimeType: "application/octet-stream",
+        providerOptions: {},
+      },
+    ];
+    const { content: ser, fileIds } = await serializeContent(
+      ctx,
+      component,
+      content
+    );
+    expect(called).toBe(true);
+    expect(fileIds).toEqual(["file-123"]);
+    // Should have replaced data with a URL
+    const serArr = ser as SerializedContent;
+    expect(typeof (serArr as { data: unknown }[])[0].data).toBe("string");
+    expect((serArr as { data: unknown }[])[0].data as string).toMatch(
+      /^https?:\/\//
+    );
+  });
+
+  test("sanity: fileIds are not returned for small files", async () => {
+    const arr = new Uint8Array([1, 2, 3, 4, 5]);
+    const ab = arr.buffer.slice(
+      arr.byteOffset,
+      arr.byteOffset + arr.byteLength
+    );
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async () => ({
+        fileId: "file-123",
+        storageId: "storage-123",
+      }),
+      storage: {
+        store: async () => "storageId",
+        getUrl: async () => "https://example.com/file",
+        delete: async () => undefined,
+      },
+    } as unknown as ActionCtx;
+    const component = api as unknown as AgentComponent;
+    const content = [
+      {
+        type: "file" as const,
+        data: ab,
+        filename: "smallfile.bin",
+        mimeType: "application/octet-stream",
+        providerOptions: {},
+      },
+    ];
+    const { fileIds } = await serializeContent(ctx, component, content);
+    expect(fileIds).toBeUndefined();
+  });
+});

package/src/mapping.ts

@@ -6,18 +6,20 @@ import {
   type AssistantContent,
   type CoreMessage,
   type DataContent,
+  type FilePart,
   type GenerateObjectResult,
+  type ImagePart,
   type StepResult,
   type ToolContent,
   type ToolSet,
   type UserContent,
 } from "ai";
 import { assert } from "convex-helpers";
-import type {
-  MessageWithMetadata,
-  Step,
-  StepWithMessagesWithMetadata,
-} from "./validators";
+import type { MessageWithMetadata } from "./validators";
+import type { ActionCtx, AgentComponent } from "./client/types.js";
+import type { RunMutationCtx } from "./client/types.js";
+import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
+
 
 export type AIMessageWithoutId = Omit<AIMessage, "id">;
 
@@ -37,23 +39,26 @@ export type SerializedContent = SerializeUrlsAndUint8Arrays<Content>;
 
 export type SerializedMessage = SerializeUrlsAndUint8Arrays<CoreMessage>;
 
-export function serializeMessage(
+export async function serializeMessage(
+  ctx: ActionCtx | RunMutationCtx,
+  component: AgentComponent,
   messageWithId: CoreMessage & { id?: string }
-): SerializedMessage {
+): Promise<{ message: SerializedMessage; fileIds?: string[] }> {
   const { id: _, experimental_providerMetadata, ...message } = messageWithId;
-  const content = message.content;
+  const { content, fileIds } = await serializeContent(
+    ctx,
+    component,
+    message.content
+  );
   return {
-    // for backwards compatibility
-    providerOptions: experimental_providerMetadata,
-    ...message,
-    content: serializeContent(content),
-  } as SerializedMessage;
-}
-
-export function serializeMessageWithId(
-  messageWithId: CoreMessage & { id?: string }
-): { message: SerializedMessage; id: string | undefined } {
-  return { message: serializeMessage(messageWithId), id: messageWithId.id };
+    message: {
+      // for backwards compatibility
+      providerOptions: experimental_providerMetadata,
+      ...message,
+      content,
+    } as SerializedMessage,
+    fileIds,
+  };
 }
 
 export function deserializeMessage(message: SerializedMessage): CoreMessage {
@@ -63,29 +68,12 @@ export function deserializeMessage(message: SerializedMessage): CoreMessage {
   } as CoreMessage;
 }
 
-export function serializeStep<TOOLS extends ToolSet>(
-  step: StepResult<TOOLS>
-): Step {
-  const content = step.response?.messages.map((message) => {
-    return serializeMessageWithId(message);
-  });
-  const timestamp = step.response?.timestamp.getTime();
-  const response = {
-    ...step.response,
-    messages: content,
-    timestamp,
-    headers: {}, // these are large and low value
-  };
-  return {
-    ...step,
-    response,
-  };
-}
-
-export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
+export async function serializeNewMessagesInStep<TOOLS extends ToolSet>(
+  ctx: ActionCtx,
+  component: AgentComponent,
   step: StepResult<TOOLS>,
   metadata: { model: string; provider: string }
-): MessageWithMetadata[] {
+): Promise<MessageWithMetadata[]> {
   // If there are tool results, there's another message with the tool results
   // ref: https://github.com/vercel/ai/blob/main/packages/ai/core/generate-text/to-response-messages.ts
   const assistantFields = {
@@ -101,24 +89,25 @@ export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
   const toolFields = {
     sources: step.sources,
   };
-  const messages: MessageWithMetadata[] = (
-    step.toolResults.length > 0
+  const messages: MessageWithMetadata[] = await Promise.all(
+    (step.toolResults.length > 0
       ? step.response.messages.slice(-2)
       : step.response.messages.slice(-1)
-  ).map(
-    (message): MessageWithMetadata => ({
-      message: serializeMessage(message),
-      // Let's not store the ID by default here. It's being generated internally
-      // and not referenced elsewhere that we know of.
-      // id: message.id,
-      ...(message.role === "tool" ? toolFields : assistantFields),
-      text: step.text,
-      // fileId: message.fileId,
-      files: step.files.map((file) => ({
-        mimeType: file.mimeType,
-        data: serializeDataOrUrl(file.uint8Array ?? file.base64),
-        // TODO: if the file is big, store it and populate url, fileId
-      })),
+    ).map(async (messageWithId): Promise<MessageWithMetadata> => {
+      const { message, fileIds } = await serializeMessage(
+        ctx,
+        component,
+        messageWithId
+      );
+      return {
+        message,
+        // Let's not store the ID by default here. It's being generated internally
+        // and not referenced elsewhere that we know of.
+        // id: message.id,
+        ...(message.role === "tool" ? toolFields : assistantFields),
+        text: step.text,
+        fileIds,
+      };
     })
   );
   return messages;
@@ -127,13 +116,18 @@ export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
 export function serializeObjectResult(
   result: GenerateObjectResult<unknown>,
   metadata: { model: string; provider: string }
-): StepWithMessagesWithMetadata {
+): { messages: MessageWithMetadata[] } {
   const text = JSON.stringify(result.object);
 
+  const message = {
+    role: "assistant" as const,
+    content: text,
+    id: result.response.id,
+  };
   return {
     messages: [
       {
-        message: { role: "assistant" as const, content: text },
+        message,
         id: result.response.id,
         model: metadata.model,
         provider: metadata.provider,
@@ -144,50 +138,60 @@ export function serializeObjectResult(
         warnings: result.warnings,
       },
     ],
-    step: {
-      text,
-      isContinued: false,
-      stepType: "initial",
-      toolCalls: [],
-      toolResults: [],
-      usage: result.usage,
-      warnings: result.warnings,
-      finishReason: result.finishReason,
-      providerMetadata: result.providerMetadata,
-      request: result.request,
-      response: {
-        ...result.response,
-        timestamp: result.response.timestamp.getTime(),
-        messages: [
-          serializeMessageWithId({
-            role: "assistant" as const,
-            content: text,
-            id: result.response.id,
-          }),
-        ],
-      },
-    },
   };
 }
 
-export function serializeContent(content: Content): SerializedContent {
+export async function serializeContent(
+  ctx: ActionCtx | RunMutationCtx,
+  component: AgentComponent,
+  content: Content
+): Promise<{ content: SerializedContent; fileIds?: string[] }> {
   if (typeof content === "string") {
-    return content;
+    return { content };
   }
-  const serialized = content.map(
-    ({ experimental_providerMetadata, ...rest }) => {
+  const fileIds: string[] = [];
+  const serialized = await Promise.all(
+    content.map(async ({ experimental_providerMetadata, ...rest }) => {
       const part = { providerOptions: experimental_providerMetadata, ...rest };
       switch (part.type) {
-        case "image":
-          return { ...part, image: serializeDataOrUrl(part.image) };
-        case "file":
-          return { ...part, data: serializeDataOrUrl(part.data) };
+        case "image": {
+          let image = serializeDataOrUrl(part.image);
+          if (
+            image instanceof ArrayBuffer &&
+            image.byteLength > MAX_FILE_SIZE
+          ) {
+            const { file } = await storeFile(
+              ctx,
+              component,
+              new Blob([image], { type: part.mimeType || guessMimeType(image) })
+            );
+            image = file.url;
+            fileIds.push(file.fileId);
+          }
+          return { ...part, image };
+        }
+        case "file": {
+          let data = serializeDataOrUrl(part.data);
+          if (data instanceof ArrayBuffer && data.byteLength > MAX_FILE_SIZE) {
+            const { file } = await storeFile(
+              ctx,
+              component,
+              new Blob([data], { type: part.mimeType })
+            );
+            data = file.url;
+            fileIds.push(file.fileId);
+          }
+          return { ...part, data };
+        }
         default:
           return part;
       }
-    }
+    })
   );
-  return serialized as SerializedContent;
+  return {
+    content: serialized as SerializedContent,
+    fileIds: fileIds.length > 0 ? fileIds : undefined,
+  };
 }
 
 export function deserializeContent(content: SerializedContent): Content {
@@ -206,8 +210,74 @@ export function deserializeContent(content: SerializedContent): Content {
   }) as Content;
 }
 
-// TODO: store in file storage if it's big
-function serializeDataOrUrl(
+/**
+ * Return a best-guess MIME type based on the magic-number signature
+ * found at the start of an ArrayBuffer.
+ *
+ * @param buf – the source ArrayBuffer
+ * @returns the detected MIME type, or `"application/octet-stream"` if unknown
+ */
+export function guessMimeType(buf: ArrayBuffer | string): string {
+  if (typeof buf === "string") {
+    if (buf.match(/^data:\w+\/\w+;base64/)) {
+      return buf.split(";")[0].split(":")[1]!;
+    }
+    return "text/plain";
+  }
+  if (buf.byteLength < 4) return "application/octet-stream";
+
+  // Read the first 12 bytes (enough for all signatures below)
+  const bytes = new Uint8Array(buf.slice(0, 12));
+  const hex = [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("");
+
+  // Helper so we can look at only the needed prefix
+  const startsWith = (sig: string) => hex.startsWith(sig.toLowerCase());
+
+  // --- image formats ---
+  if (startsWith("89504e47")) return "image/png"; // PNG  - 89 50 4E 47
+  if (
+    startsWith("ffd8ffdb") ||
+    startsWith("ffd8ffe0") ||
+    startsWith("ffd8ffee") ||
+    startsWith("ffd8ffe1")
+  )
+    return "image/jpeg"; // JPEG
+  if (startsWith("47494638")) return "image/gif"; // GIF
+  if (startsWith("424d")) return "image/bmp"; // BMP
+  if (startsWith("52494646") && hex.substr(16, 8) === "57454250")
+    return "image/webp"; // WEBP (RIFF....WEBP)
+  if (startsWith("49492a00")) return "image/tiff"; // TIFF
+  // <svg in hex is 3c 3f 78 6d 6c
+  if (startsWith("3c737667")) return "image/svg+xml"; // <svg
+  if (startsWith("3c3f786d")) return "image/svg+xml"; // <?xm
+
+  // --- audio/video ---
+  if (startsWith("494433")) return "audio/mpeg"; // MP3 (ID3)
+  if (startsWith("000001ba") || startsWith("000001b3")) return "video/mpeg"; // MPEG container
+  if (startsWith("1a45dfa3")) return "video/webm"; // WEBM / Matroska
+  if (startsWith("00000018") && hex.substr(16, 8) === "66747970")
+    return "video/mp4"; // MP4
+  if (startsWith("4f676753")) return "audio/ogg"; // OGG / Opus
+
+  // --- documents & archives ---
+  if (startsWith("25504446")) return "application/pdf"; // PDF
+  if (
+    startsWith("504b0304") ||
+    startsWith("504b0506") ||
+    startsWith("504b0708")
+  )
+    return "application/zip"; // ZIP / DOCX / PPTX / XLSX / EPUB
+  if (startsWith("52617221")) return "application/x-rar-compressed"; // RAR
+  if (startsWith("7f454c46")) return "application/x-elf"; // ELF binaries
+  if (startsWith("1f8b08")) return "application/gzip"; // GZIP
+  if (startsWith("425a68")) return "application/x-bzip2"; // BZIP2
+  if (startsWith("3c3f786d6c")) return "application/xml"; // XML
+
+  // Plain text, JSON and others are trickier—fallback:
+  return "application/octet-stream";
+}
+
+export function serializeDataOrUrl(
   dataOrUrl: DataContent | URL
 ): ArrayBuffer | string {
   if (typeof dataOrUrl === "string") {
@@ -225,7 +295,9 @@ function serializeDataOrUrl(
   ) as ArrayBuffer;
 }
 
-function deserializeUrl(urlOrString: string | ArrayBuffer): URL | DataContent {
+export function deserializeUrl(
+  urlOrString: string | ArrayBuffer
+): URL | DataContent {
   if (typeof urlOrString === "string") {
     if (
       urlOrString.startsWith("http://") ||
@@ -238,18 +310,16 @@ function deserializeUrl(urlOrString: string | ArrayBuffer): URL | DataContent {
   return urlOrString;
 }
 
-export function toUIFilePart(file: {
-  data?: ArrayBuffer | string;
-  url?: string;
-  mimeType: string;
-}): FileUIPart {
+export function toUIFilePart(part: ImagePart | FilePart): FileUIPart {
+  const dataOrUrl = serializeDataOrUrl(
+    part.type === "image" ? part.image : part.data
+  );
+
   return {
     type: "file",
     data:
-      file.data instanceof ArrayBuffer
-        ? encodeBase64(file.data)
-        : file.url ?? file.data ?? "",
-    mimeType: file.mimeType,
+      dataOrUrl instanceof ArrayBuffer ? encodeBase64(dataOrUrl) : dataOrUrl,
+    mimeType: part.mimeType ?? guessMimeType(dataOrUrl),
   };
 }
 

package/src/react/deltas.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from "vitest";
 import { mergeDeltas, applyDeltasToStreamMessage } from "./deltas";
 import type { StreamMessage, StreamDelta, TextStreamPart } from "../validators";
+import { omit } from "convex-helpers";
 
 function makeStreamMessage(
   streamId: string,
@@ -283,7 +284,9 @@ describe("mergeDeltas", () => {
       [],
       deltas
     );
-    expect(messages1).toEqual(messages2);
+    expect(messages1.map((m) => omit(m, ["_creationTime"]))).toEqual(
+      messages2.map((m) => omit(m, ["_creationTime"]))
+    );
     expect(streams1).toEqual(streams2);
     expect(changed1).toBe(changed2);
     // Inputs should remain unchanged

package/src/react/toUIMessages.ts

@@ -43,9 +43,14 @@ export function toUIMessages(
       if (text) {
         parts.push({ type: "text", text });
       }
-      if (message.files) {
-        parts.push(...message.files.map(toUIFilePart));
-      }
+      nonStringContent.forEach((contentPart) => {
+        switch (contentPart.type) {
+          case "file":
+          case "image":
+            parts.push(toUIFilePart(contentPart));
+            break;
+        }
+      });
       uiMessages.push({
         ...common,
         role: "user",
@@ -95,11 +100,12 @@ export function toUIMessages(
           source,
         });
       }
-      for (const file of message.files ?? []) {
-        assistantMessage.parts.push(toUIFilePart(file));
-      }
       for (const contentPart of nonStringContent) {
         switch (contentPart.type) {
+          case "file":
+          case "image":
+            assistantMessage.parts.push(toUIFilePart(contentPart));
+            break;
           case "tool-call":
             assistantMessage.parts.push({
               type: "step-start",