@convex-dev/agent 0.1.18 → 0.2.0

package/src/client/definePlaygroundAPI.ts

@@ -1,27 +1,27 @@
 import {
+  actionGeneric,
+  mutationGeneric,
   paginationOptsValidator,
   queryGeneric,
-  mutationGeneric,
-  actionGeneric,
-  type GenericDataModel,
-  type GenericQueryCtx,
   type ApiFromModules,
   type GenericActionCtx,
+  type GenericDataModel,
+  type GenericQueryCtx,
 } from "convex/server";
+import { v } from "convex/values";
 import {
+  createThread as createThread_,
+  listMessages as listMessages_,
+  deserializeMessage,
+  vContextOptions,
+  vMessage,
   vMessageDoc,
-  vThreadDoc,
   vPaginationResult,
-  vMessage,
-  vContextOptions,
   vStorageOptions,
-  type AgentComponent,
+  vThreadDoc,
   type Agent,
-  listMessages as listMessages_,
-  createThread as createThread_,
+  type AgentComponent,
 } from "./index.js";
-import { v } from "convex/values";
-import { deserializeMessage } from "../mapping.js";
 
 export type PlaygroundAPI = ApiFromModules<{
   playground: ReturnType<typeof definePlaygroundAPI>;
@@ -61,9 +61,7 @@ export function definePlaygroundAPI<DataModel extends GenericDataModel>(
   }
 
   const isApiKeyValid = queryGeneric({
-    args: {
-      apiKey: v.string(),
-    },
+    args: { apiKey: v.string() },
     handler: async (ctx, args) => {
       try {
         await validateApiKey(ctx, args.apiKey);
@@ -107,18 +105,14 @@ export function definePlaygroundAPI<DataModel extends GenericDataModel>(
         instructions: agent.options.instructions,
         contextOptions: agent.options.contextOptions,
         storageOptions: agent.options.storageOptions,
-        maxSteps: agent.options.maxSteps,
-        maxRetries: agent.options.maxRetries,
+        maxRetries: agent.options.callSettings?.maxRetries,
         tools: agent.options.tools ? Object.keys(agent.options.tools) : [],
       }));
     },
   });
 
   const listUsers = queryGeneric({
-    args: {
-      apiKey: v.string(),
-      paginationOpts: paginationOptsValidator,
-    },
+    args: { apiKey: v.string(), paginationOpts: paginationOptsValidator },
     handler: async (ctx, args) => {
       await validateApiKey(ctx, args.apiKey);
       const users = await ctx.runQuery(component.users.listUsersWithThreads, {
@@ -134,12 +128,7 @@ export function definePlaygroundAPI<DataModel extends GenericDataModel>(
         ),
       };
     },
-    returns: vPaginationResult(
-      v.object({
-        _id: v.string(),
-        name: v.string(),
-      }),
-    ),
+    returns: vPaginationResult(v.object({ _id: v.string(), name: v.string() })),
   });
 
   // List threads for a user (query)
@@ -168,10 +157,7 @@ export function definePlaygroundAPI<DataModel extends GenericDataModel>(
             } = await ctx.runQuery(component.messages.listMessagesByThreadId, {
               threadId: thread._id,
               order: "desc",
-              paginationOpts: {
-                numItems: 1,
-                cursor: null,
-              },
+              paginationOpts: { numItems: 1, cursor: null },
             });
             return {
               ...thread,

package/src/client/files.ts

@@ -1,4 +1,10 @@
-import type { FilePart, ImagePart } from "ai";
+import type {
+  AssistantContent,
+  FilePart,
+  ImagePart,
+  ModelMessage,
+  UserContent,
+} from "ai";
 import type { Id } from "../component/_generated/dataModel.js";
 import type {
   ActionCtx,
@@ -6,6 +12,8 @@ import type {
   QueryCtx,
   RunMutationCtx,
 } from "./types.js";
+import type { Message } from "../validators.js";
+import { assert } from "convex-helpers";
 
 export const MAX_FILE_SIZE = 1024 * 64;
 
@@ -22,8 +30,8 @@ type File = {
  * @param ctx A ctx object from an action.
  * @param component The agent component.
  * @param blob The blob to store.
- * @param filename The filename to store.
- * @param sha256 The sha256 hash of the file. If not provided, it will be
+ * @param args.filename The filename to store.
+ * @param args.sha256 The sha256 hash of the file. If not provided, it will be
  *   computed. However, to ensure no corruption during transfer, you can
  *   calculate this on the client to enforce integrity.
  * @returns The URL, fileId, and storageId of the stored file.
@@ -32,8 +40,7 @@ export async function storeFile(
   ctx: ActionCtx | RunMutationCtx,
   component: AgentComponent,
   blob: Blob,
-  filename?: string,
-  sha256?: string,
+  { filename, sha256 }: { filename?: string; sha256?: string } = {},
 ): Promise<{
   file: File;
   filePart: FilePart;
@@ -148,24 +155,94 @@ export async function getFile(
 
 function getParts(
   url: string,
-  mimeType: string,
+  mediaType: string,
   filename: string | undefined,
-): {
-  filePart: FilePart;
-  imagePart: ImagePart | undefined;
-} {
+): { filePart: FilePart; imagePart: ImagePart | undefined } {
   const filePart: FilePart = {
     type: "file",
     data: new URL(url),
-    mimeType,
+    mediaType,
     filename,
   };
-  const imagePart: ImagePart | undefined = mimeType.startsWith("image/")
-    ? {
-        type: "image",
-        image: new URL(url),
-        mimeType,
-      }
+  const imagePart: ImagePart | undefined = mediaType.startsWith("image/")
+    ? { type: "image", image: new URL(url), mediaType }
     : undefined;
   return { filePart, imagePart };
 }
+
+/**
+ * Check if a URL points to localhost
+ */
+function isLocalhostUrl(url: URL): boolean {
+  return (
+    url.hostname === "localhost" ||
+    url.hostname === "127.0.0.1" ||
+    url.hostname === "::1" ||
+    url.hostname === "0.0.0.0"
+  );
+}
+
+/**
+ * Download a file from a URL
+ */
+async function downloadFile(url: URL): Promise<ArrayBuffer> {
+  // Fetch the file
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+  }
+
+  return await response.arrayBuffer();
+}
+
+/**
+ * Process messages to inline file and image URLs that point to localhost
+ * by converting them to base64. This solves the problem of LLMs not being
+ * able to access localhost URLs.
+ */
+export async function inlineMessagesFiles(
+  messages: (ModelMessage | Message)[],
+): Promise<(ModelMessage | Message)[]> {
+  // Process each message to convert localhost URLs to base64
+  return Promise.all(
+    messages.map(async (message): Promise<ModelMessage | Message> => {
+      if (
+        (message.role !== "user" && message.role !== "assistant") ||
+        typeof message.content === "string" ||
+        !Array.isArray(message.content)
+      ) {
+        return message;
+      }
+
+      const processedContent = await Promise.all(
+        message.content.map(async (part) => {
+          if (part.type === "image" && part.image instanceof URL) {
+            assert(
+              message.role === "user",
+              "Images can only be in user messages",
+            );
+            if (isLocalhostUrl(part.image)) {
+              const imageData = await downloadFile(part.image);
+              return { ...part, image: imageData } as ImagePart;
+            }
+          }
+
+          // Handle file parts
+          if (part.type === "file" && part.data instanceof URL) {
+            if (isLocalhostUrl(part.data)) {
+              const fileData = await downloadFile(part.data);
+              return { ...part, data: fileData } as FilePart;
+            }
+          }
+
+          return part;
+        }),
+      );
+      if (message.role === "user") {
+        return { ...message, content: processedContent as UserContent };
+      } else {
+        return { ...message, content: processedContent as AssistantContent };
+      }
+    }),
+  );
+}

package/src/client/index.test.ts

@@ -19,11 +19,14 @@ import type {
 } from "convex/server";
 import { v } from "convex/values";
 import { defineSchema } from "convex/server";
-import { MockLanguageModelV1 } from "ai/test";
-import type { LanguageModelV1, LanguageModelV1StreamPart } from "ai";
-import { simulateReadableStream } from "ai";
+import { MockLanguageModelV2 } from "ai/test";
+import type {
+  LanguageModelV2,
+  LanguageModelV2StreamPart,
+} from "@ai-sdk/provider";
+import { simulateReadableStream, stepCountIs } from "ai";
 import { components, initConvexTest } from "./setup.test.js";
-import { z } from "zod";
+import { z } from "zod/v4";
 
 const schema = defineSchema({});
 type DataModel = DataModelFromSchemaDefinition<typeof schema>;
@@ -36,7 +39,7 @@ const agent = new Agent(components.agent, {
   name: "test",
   instructions: "You are a test agent",
   // TODO: get mock model that works in v8
-  chat: mockModel(),
+  languageModel: mockModel(),
 });
 
 export const testQuery = query({
@@ -44,10 +47,7 @@ export const testQuery = query({
   handler: async (ctx, args) => {
     return await agent.listMessages(ctx, {
       threadId: args.threadId,
-      paginationOpts: {
-        cursor: null,
-        numItems: 10,
-      },
+      paginationOpts: { cursor: null, numItems: 10 },
       excludeToolMessages: true,
       statuses: ["success"],
     });
@@ -57,18 +57,14 @@ export const testQuery = query({
 export const createThread = mutation({
   args: {},
   handler: async (ctx) => {
-    const { threadId } = await agent.createThread(ctx, {
-      userId: "1",
-    });
+    const { threadId } = await agent.createThread(ctx, { userId: "1" });
     return { threadId };
   },
 });
 
 export const createThreadMutation = agent.createThreadMutation();
 export const generateObjectAction = agent.asObjectAction({
-  schema: z.object({
-    prompt: z.any().describe("The prompt passed in"),
-  }),
+  schema: z.object({ prompt: z.any().describe("The prompt passed in") }),
 });
 export const generateTextAction = agent.asTextAction({});
 export const streamTextAction = agent.asTextAction({ stream: true });
@@ -77,9 +73,7 @@ export const saveMessageMutation = agent.asSaveMessagesMutation();
 export const createAndGenerate = action({
   args: {},
   handler: async (ctx) => {
-    const { thread } = await agent.createThread(ctx, {
-      userId: "1",
-    });
+    const { thread } = await agent.createThread(ctx, { userId: "1" });
     const result = await thread.generateText({
       messages: [{ role: "user", content: "Hello" }],
     });
@@ -233,10 +227,7 @@ describe("filterOutOrphanedToolMessages", () => {
     order: 1,
     stepOrder: 2,
     tool: true,
-    message: {
-      role: "assistant",
-      content: [{ type: "text", text: "Hello" }],
-    },
+    message: { role: "assistant", content: [{ type: "text", text: "Hello" }] },
     status: "success",
     threadId: "1",
   };
@@ -252,16 +243,16 @@ describe("filterOutOrphanedToolMessages", () => {
   });
 });
 
-function mockModel(): LanguageModelV1 {
-  return new MockLanguageModelV1({
+function mockModel(): LanguageModelV2 {
+  return new MockLanguageModelV2({
     provider: "mock",
     modelId: "mock",
-    defaultObjectGenerationMode: "json",
     // supportsStructuredOutputs: true,
     doGenerate: async ({ prompt }) => ({
       finishReason: "stop",
-      usage: { completionTokens: 10, promptTokens: 3 },
-      logprobs: undefined,
+      content: [{ type: "text", text: JSON.stringify({ prompt }) }],
+      warnings: [],
+      usage: { outputTokens: 10, inputTokens: 3, totalTokens: 13 },
       rawCall: { rawPrompt: null, rawSettings: {} },
       text: JSON.stringify({ prompt }),
     }),
@@ -277,10 +268,9 @@ function mockModel(): LanguageModelV1 {
           {
             type: "finish",
             finishReason: "stop",
-            logprobs: undefined,
-            usage: { completionTokens: 10, promptTokens: 3 },
+            usage: { outputTokens: 10, inputTokens: 3, totalTokens: 13 },
           },
-        ] as LanguageModelV1StreamPart[],
+        ] as LanguageModelV2StreamPart[],
       }),
       rawCall: { rawPrompt: null, rawSettings: {} },
     }),
@@ -289,19 +279,19 @@ function mockModel(): LanguageModelV1 {
 
 describe("Agent option variations and normal behavior", () => {
   test("Agent can be constructed with minimal options", () => {
-    const a = new Agent(components.agent, { chat: mockModel() });
+    const a = new Agent(components.agent, { languageModel: mockModel() });
     expect(a).toBeInstanceOf(Agent);
   });
 
   test("Agent can be constructed with all options", () => {
     const a = new Agent(components.agent, {
       name: "full",
-      chat: mockModel(),
+      languageModel: mockModel(),
       instructions: "Test instructions",
       contextOptions: { recentMessages: 5 },
       storageOptions: { saveMessages: "all" },
-      maxSteps: 2,
-      maxRetries: 1,
+      stopWhen: stepCountIs(2),
+      callSettings: { maxRetries: 1 },
       usageHandler: async () => {},
       rawRequestResponseHandler: async () => {},
     });
@@ -346,7 +336,7 @@ describe("Agent message operations", () => {
     );
     expect(messageId).toBeTypeOf("string");
 
-    const { lastMessageId, messages } = await t.run(async (ctx) =>
+    const { messages } = await t.run(async (ctx) =>
       agent.saveMessages(ctx, {
         threadId,
         userId: "4",
@@ -357,7 +347,7 @@ describe("Agent message operations", () => {
       }),
     );
     expect(messages.length).toBe(2);
-    expect(lastMessageId).toBe(messages[1]._id);
+    expect(messages[1]._id).toBeDefined();
   });
 });
 
@@ -433,8 +423,8 @@ describe("Agent-generated mutations/actions/queries", () => {
         },
       ],
     });
-    expect(result.lastMessageId).toBeDefined();
-    expect(result.messageIds.length).toBe(1);
+    expect(result.messages.length).toBe(1);
+    expect(result.messages[0]._id).toBeDefined();
   });
 });