@convex-dev/agent 0.5.0-alpha.1 → 0.6.0-alpha.0

package/src/client/files.ts

@@ -1,11 +1,20 @@
-import type { FilePart, ImagePart } from "ai";
+import type {
+  AssistantContent,
+  FilePart,
+  ImagePart,
+  ModelMessage,
+  UserContent,
+} from "ai";
 import type { Id } from "../component/_generated/dataModel.js";
 import type {
   ActionCtx,
   AgentComponent,
+  MutationCtx,
   QueryCtx,
-  RunMutationCtx,
 } from "./types.js";
+import type { Message } from "../validators.js";
+import { assert } from "convex-helpers";
+import type { StorageReader } from "convex/server";
 
 export const MAX_FILE_SIZE = 1024 * 64;
 
@@ -22,18 +31,17 @@ type File = {
  * @param ctx A ctx object from an action.
  * @param component The agent component.
  * @param blob The blob to store.
- * @param filename The filename to store.
- * @param sha256 The sha256 hash of the file. If not provided, it will be
+ * @param args.filename The filename to store.
+ * @param args.sha256 The sha256 hash of the file. If not provided, it will be
  *   computed. However, to ensure no corruption during transfer, you can
  *   calculate this on the client to enforce integrity.
  * @returns The URL, fileId, and storageId of the stored file.
  */
 export async function storeFile(
-  ctx: ActionCtx | RunMutationCtx,
+  ctx: ActionCtx | MutationCtx,
   component: AgentComponent,
   blob: Blob,
-  filename?: string,
-  sha256?: string,
+  { filename, sha256 }: { filename?: string; sha256?: string } = {},
 ): Promise<{
   file: File;
   filePart: FilePart;
@@ -84,7 +92,7 @@ export async function storeFile(
     storageId: newStorageId,
     hash,
     filename,
-    mimeType: blob.type,
+    mediaType: blob.type,
   });
   const url = (await ctx.storage.getUrl(storageId as Id<"_storage">))!;
   if (storageId !== newStorageId) {
@@ -108,7 +116,7 @@ export async function storeFile(
 /**
  * Get file metadata from the component.
  * This also returns filePart (and imagePart if the file is an image),
- * which are useful to construct a CoreMessage like
+ * which are useful to construct a ModelMessage like
  * ```ts
  * const { filePart, imagePart } = await getFile(ctx, components.agent, fileId);
  * const message: UserMessage = {
@@ -122,7 +130,7 @@ export async function storeFile(
  * @returns The file metadata and content parts.
  */
 export async function getFile(
-  ctx: ActionCtx | QueryCtx,
+  ctx: ActionCtx | (QueryCtx & { storage: StorageReader }),
   component: AgentComponent,
   fileId: string,
 ) {
@@ -134,8 +142,10 @@ export async function getFile(
   if (!url) {
     throw new Error(`File not found in storage: ${file.storageId}`);
   }
+  // Support both mediaType (preferred) and mimeType (deprecated)
+  const mediaType = file.mediaType ?? file.mimeType ?? "";
   return {
-    ...getParts(url, file.mimeType, file.filename),
+    ...getParts(url, mediaType, file.filename),
     file: {
       fileId,
       url,
@@ -150,10 +160,7 @@ function getParts(
   url: string,
   mediaType: string,
   filename: string | undefined,
-): {
-  filePart: FilePart;
-  imagePart: ImagePart | undefined;
-} {
+): { filePart: FilePart; imagePart: ImagePart | undefined } {
   const filePart: FilePart = {
     type: "file",
     data: new URL(url),
@@ -161,11 +168,84 @@ function getParts(
     filename,
   };
   const imagePart: ImagePart | undefined = mediaType.startsWith("image/")
-    ? {
-        type: "image",
-        image: new URL(url),
-        mediaType,
-      }
+    ? { type: "image", image: new URL(url), mediaType }
     : undefined;
   return { filePart, imagePart };
 }
+
+/**
+ * Check if a URL points to localhost
+ */
+function isLocalhostUrl(url: URL): boolean {
+  return (
+    url.hostname === "localhost" ||
+    url.hostname === "127.0.0.1" ||
+    url.hostname === "::1" ||
+    url.hostname === "0.0.0.0"
+  );
+}
+
+/**
+ * Download a file from a URL
+ */
+async function downloadFile(url: URL): Promise<ArrayBuffer> {
+  // Fetch the file
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+  }
+
+  return await response.arrayBuffer();
+}
+
+/**
+ * Process messages to inline file and image URLs that point to localhost
+ * by converting them to base64. This solves the problem of LLMs not being
+ * able to access localhost URLs.
+ */
+export async function inlineMessagesFiles<T extends ModelMessage | Message>(
+  messages: T[],
+): Promise<T[]> {
+  // Process each message to convert localhost URLs to base64
+  return Promise.all(
+    messages.map(async (message): Promise<T> => {
+      if (
+        (message.role !== "user" && message.role !== "assistant") ||
+        typeof message.content === "string" ||
+        !Array.isArray(message.content)
+      ) {
+        return message;
+      }
+
+      const processedContent = await Promise.all(
+        message.content.map(async (part) => {
+          if (part.type === "image" && part.image instanceof URL) {
+            assert(
+              message.role === "user",
+              "Images can only be in user messages",
+            );
+            if (isLocalhostUrl(part.image)) {
+              const imageData = await downloadFile(part.image);
+              return { ...part, image: imageData } as ImagePart;
+            }
+          }
+
+          // Handle file parts
+          if (part.type === "file" && part.data instanceof URL) {
+            if (isLocalhostUrl(part.data)) {
+              const fileData = await downloadFile(part.data);
+              return { ...part, data: fileData } as FilePart;
+            }
+          }
+
+          return part;
+        }),
+      );
+      if (message.role === "user") {
+        return { ...message, content: processedContent as UserContent };
+      } else {
+        return { ...message, content: processedContent as AssistantContent };
+      }
+    }),
+  );
+}

package/src/client/index.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "vitest";
 import {
   Agent,
+  createThread,
   filterOutOrphanedToolMessages,
   type MessageDoc,
 } from "./index.js";
@@ -19,14 +20,10 @@ import type {
 } from "convex/server";
 import { v } from "convex/values";
 import { defineSchema } from "convex/server";
-import { MockLanguageModelV2 } from "ai/test";
-import type {
-  LanguageModelV2,
-  LanguageModelV2StreamPart,
-} from "@ai-sdk/provider";
-import { simulateReadableStream, stepCountIs } from "ai";
+import { stepCountIs } from "ai";
 import { components, initConvexTest } from "./setup.test.js";
-import { z } from "zod";
+import { z } from "zod/v4";
+import { mockModel } from "./mockModel.js";
 
 const schema = defineSchema({});
 type DataModel = DataModelFromSchemaDefinition<typeof schema>;
@@ -35,11 +32,14 @@ const query = queryGeneric as QueryBuilder<DataModel, "public">;
 const mutation = mutationGeneric as MutationBuilder<DataModel, "public">;
 const action = actionGeneric as ActionBuilder<DataModel, "public">;
 
+const TEST_TEXT = JSON.stringify({ hello: "world" });
+
 const agent = new Agent(components.agent, {
   name: "test",
   instructions: "You are a test agent",
-  // TODO: get mock model that works in v8
-  chat: mockModel(),
+  languageModel: mockModel({
+    content: [{ type: "text", text: TEST_TEXT }],
+  }),
 });
 
 export const testQuery = query({
@@ -47,31 +47,24 @@ export const testQuery = query({
   handler: async (ctx, args) => {
     return await agent.listMessages(ctx, {
       threadId: args.threadId,
-      paginationOpts: {
-        cursor: null,
-        numItems: 10,
-      },
+      paginationOpts: { cursor: null, numItems: 10 },
       excludeToolMessages: true,
       statuses: ["success"],
     });
   },
 });
 
-export const createThread = mutation({
+export const createThreadManually = mutation({
   args: {},
   handler: async (ctx) => {
-    const { threadId } = await agent.createThread(ctx, {
-      userId: "1",
-    });
+    const { threadId } = await agent.createThread(ctx, { userId: "1" });
     return { threadId };
   },
 });
 
 export const createThreadMutation = agent.createThreadMutation();
 export const generateObjectAction = agent.asObjectAction({
-  schema: z.object({
-    prompt: z.any().describe("The prompt passed in"),
-  }),
+  schema: z.object({ hello: z.string().describe("A string for testing") }),
 });
 export const generateTextAction = agent.asTextAction({});
 export const streamTextAction = agent.asTextAction({ stream: true });
@@ -80,9 +73,7 @@ export const saveMessageMutation = agent.asSaveMessagesMutation();
 export const createAndGenerate = action({
   args: {},
   handler: async (ctx) => {
-    const { thread } = await agent.createThread(ctx, {
-      userId: "1",
-    });
+    const { thread } = await agent.createThread(ctx, { userId: "1" });
     const result = await thread.generateText({
       messages: [{ role: "user", content: "Hello" }],
     });
@@ -162,7 +153,7 @@ export const fetchContextAction = action({
 const testApi: ApiFromModules<{
   fns: {
     createAndGenerate: typeof createAndGenerate;
-    createThread: typeof createThread;
+    createThreadManually: typeof createThreadManually;
     testQuery: typeof testQuery;
     continueThreadAction: typeof continueThreadAction;
     generateTextWithThread: typeof generateTextWithThread;
@@ -172,20 +163,19 @@ const testApi: ApiFromModules<{
     generateObjectAction: typeof generateObjectAction;
     saveMessageMutation: typeof saveMessageMutation;
   };
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
 }>["fns"] = anyApi["index.test"] as any;
 
 describe("Agent thick client", () => {
   test("should create a thread", async () => {
     const t = initConvexTest(schema);
-    const result = await t.mutation(testApi.createThread, {});
+    const result = await t.mutation(testApi.createThreadManually, {});
     expect(result.threadId).toBeTypeOf("string");
   });
   test("should create a thread and generate text", async () => {
     const t = initConvexTest(schema);
     const result = await t.action(testApi.createAndGenerate, {});
     expect(result).toBeDefined();
-    expect(result).toMatch("Hello");
+    expect(result).toMatch(TEST_TEXT);
   });
 });
 
@@ -203,6 +193,7 @@ describe("filterOutOrphanedToolMessages", () => {
           type: "tool-call",
           toolCallId: "1",
           toolName: "tool1",
+          input: { test: "test" },
           args: { test: "test" },
         },
       ],
@@ -236,10 +227,7 @@ describe("filterOutOrphanedToolMessages", () => {
     order: 1,
     stepOrder: 2,
     tool: true,
-    message: {
-      role: "assistant",
-      content: [{ type: "text", text: "Hello" }],
-    },
+    message: { role: "assistant", content: [{ type: "text", text: "Hello" }] },
     status: "success",
     threadId: "1",
   };
@@ -255,55 +243,24 @@ describe("filterOutOrphanedToolMessages", () => {
   });
 });
 
-function mockModel(): LanguageModelV2 {
-  return new MockLanguageModelV2({
-    provider: "mock",
-    modelId: "mock",
-    // supportsStructuredOutputs: true,
-    doGenerate: async ({ prompt }) => ({
-      finishReason: "stop",
-      content: [{ type: "text", text: JSON.stringify({ prompt }) }],
-      warnings: [],
-      usage: { outputTokens: 10, inputTokens: 3, totalTokens: 13 },
-      rawCall: { rawPrompt: null, rawSettings: {} },
-      text: JSON.stringify({ prompt }),
-    }),
-    doStream: async ({ prompt }) => ({
-      stream: simulateReadableStream({
-        chunkDelayInMs: 50,
-        initialDelayInMs: 100,
-        chunks: [
-          {
-            type: "text-delta",
-            textDelta: `This is a sample response to ${JSON.stringify(prompt)}`,
-          },
-          {
-            type: "finish",
-            finishReason: "stop",
-            usage: { outputTokens: 10, inputTokens: 3, totalTokens: 13 },
-          },
-        ] as LanguageModelV2StreamPart[],
-      }),
-      rawCall: { rawPrompt: null, rawSettings: {} },
-    }),
-  });
-}
-
 describe("Agent option variations and normal behavior", () => {
   test("Agent can be constructed with minimal options", () => {
-    const a = new Agent(components.agent, { chat: mockModel() });
+    const a = new Agent(components.agent, {
+      name: "minimal",
+      languageModel: mockModel(),
+    });
     expect(a).toBeInstanceOf(Agent);
   });
 
   test("Agent can be constructed with all options", () => {
     const a = new Agent(components.agent, {
       name: "full",
-      chat: mockModel(),
+      languageModel: mockModel(),
       instructions: "Test instructions",
       contextOptions: { recentMessages: 5 },
       storageOptions: { saveMessages: "all" },
       stopWhen: stepCountIs(2),
-      maxRetries: 1,
+      callSettings: { maxRetries: 1 },
       usageHandler: async () => {},
       rawRequestResponseHandler: async () => {},
     });
@@ -315,7 +272,7 @@ describe("Agent thread management", () => {
   test("createThread returns threadId (mutation context)", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "2" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "2" }),
     );
     expect(threadId).toBeTypeOf("string");
   });
@@ -323,7 +280,7 @@ describe("Agent thread management", () => {
   test("continueThread returns thread object", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "3" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "3" }),
     );
     const result = await t.action(testApi.continueThreadAction, {
       threadId,
@@ -337,7 +294,7 @@ describe("Agent message operations", () => {
   test("saveMessage and saveMessages store messages", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "4" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "4" }),
     );
     const { messageId } = await t.run(async (ctx) =>
       agent.saveMessage(ctx, {
@@ -348,7 +305,7 @@ describe("Agent message operations", () => {
     );
     expect(messageId).toBeTypeOf("string");
 
-    const { lastMessageId, messages } = await t.run(async (ctx) =>
+    const { messages } = await t.run(async (ctx) =>
       agent.saveMessages(ctx, {
         threadId,
         userId: "4",
@@ -359,7 +316,7 @@ describe("Agent message operations", () => {
       }),
     );
     expect(messages.length).toBe(2);
-    expect(lastMessageId).toBe(messages[1]._id);
+    expect(messages[1]._id).toBeDefined();
   });
 });
 
@@ -367,7 +324,7 @@ describe("Agent text/object generation", () => {
   test("generateText with custom context and storage options", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "5" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "5" }),
     );
     const result = await t.action(testApi.generateTextWithThread, {
       threadId,
@@ -376,13 +333,13 @@ describe("Agent text/object generation", () => {
       contextOptions: { recentMessages: 1 },
       storageOptions: { saveMessages: "all" },
     });
-    expect(result.text).toMatch(/Test/);
+    expect(result.text).toEqual(TEST_TEXT);
   });
 
   test("generateObject returns object", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "6" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "6" }),
     );
     const result = await t.action(testApi.generateObjectWithThread, {
       threadId,
@@ -397,21 +354,21 @@ describe("Agent-generated mutations/actions/queries", () => {
   test("createThreadMutation works via t.mutation", async () => {
     const t = initConvexTest(schema);
     // This test is for the registered mutation, not the agent method
-    const result = await t.mutation(testApi.createThread, {});
+    const result = await t.mutation(testApi.createThreadManually, {});
     expect(result.threadId).toBeTypeOf("string");
   });
 
   test("asTextAction and asObjectAction work via t.action", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "8" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "8" }),
     );
     const textResult = await t.action(testApi.generateTextAction, {
       userId: "8",
       threadId,
       messages: [{ role: "user", content: "Say hi" }],
     });
-    expect(textResult.text).toMatch(/Say hi/);
+    expect(textResult.text).toEqual(TEST_TEXT);
 
     const objResult = await t.action(testApi.generateObjectAction, {
       userId: "8",
@@ -424,7 +381,7 @@ describe("Agent-generated mutations/actions/queries", () => {
   test("asSaveMessagesMutation works via t.mutation", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent.createThread(ctx, { userId: "9" }).then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "9" }),
     );
     const result = await t.mutation(testApi.saveMessageMutation, {
       threadId,
@@ -435,8 +392,8 @@ describe("Agent-generated mutations/actions/queries", () => {
         },
       ],
     });
-    expect(result.lastMessageId).toBeDefined();
-    expect(result.messageIds.length).toBe(1);
+    expect(result.messages.length).toBe(1);
+    expect(result.messages[0]._id).toBeDefined();
   });
 });
 
@@ -444,9 +401,7 @@ describe("Agent context and search options", () => {
   test("fetchContextMessages returns context messages", async () => {
     const t = initConvexTest(schema);
     const threadId = await t.run(async (ctx) =>
-      agent
-        .createThread(ctx, { userId: "10" })
-        .then(({ threadId }) => threadId),
+      createThread(ctx, components.agent, { userId: "10" }),
     );
     await t.run(async (ctx) =>
       agent.saveMessage(ctx, {