@convex-dev/agent 0.1.8-alpha.0 → 0.1.8-alpha.1

package/package.json

@@ -7,7 +7,7 @@
     "email": "support@convex.dev",
     "url": "https://github.com/get-convex/agent/issues"
   },
-  "version": "0.1.8-alpha.0",
+  "version": "0.1.8-alpha.1",
   "license": "Apache-2.0",
   "keywords": [
     "convex",
@@ -39,8 +39,7 @@
   },
   "files": [
     "dist",
-    "src",
-    "react"
+    "src"
   ],
   "exports": {
     "./package.json": "./package.json",
@@ -91,7 +90,7 @@
   "peerDependencies": {
     "ai": "^4.3.16",
     "convex": "^1.23.0",
-    "convex-helpers": "^0.1.91",
+    "convex-helpers": "^0.1.93",
     "react": "^18.3.1 || ^19.0.0"
   },
   "peerDependenciesMeta": {
@@ -106,7 +105,7 @@
     "@types/react": "^19.1.1",
     "chokidar-cli": "^3.0.0",
     "convex": "^1.24.8",
-    "convex-helpers": "^0.1.92",
+    "convex-helpers": "^0.1.93",
     "convex-test": "^0.0.37",
     "cpy-cli": "^5.0.0",
     "eslint": "^9.24.0",

package/src/client/index.ts

@@ -2,11 +2,14 @@ import type { EmbeddingModelV1, LanguageModelV1 } from "@ai-sdk/provider";
 import type {
   CoreMessage,
   DeepPartial,
+  FilePart,
   GenerateObjectResult,
   GenerateTextResult,
+  ImagePart,
   StepResult,
   StreamObjectResult,
   StreamTextResult,
+  TextPart,
   ToolSet,
 } from "ai";
 import { generateObject, generateText, streamObject, streamText } from "ai";
@@ -1495,6 +1498,17 @@ export class Agent<AgentTools extends ToolSet> {
       order = saved.messages.at(-1)?.order;
       stepOrder = saved.messages.at(-1)?.stepOrder;
     }
+
+    let processedMessages = [
+      ...contextMessages.map((m) => deserializeMessage(m.message!)),
+      ...messages,
+    ];
+
+    // Process messages to inline localhost files (if not, file urls pointing to localhost will be sent to LLM providers)
+    if (process.env.CONVEX_CLOUD_URL?.startsWith("http://127.0.0.1")) {
+      processedMessages = await this._inlineMessagesFiles(processedMessages);
+    }
+
     const { prompt: _, model, ...rest } = args;
     return {
       args: {
@@ -1502,10 +1516,7 @@ export class Agent<AgentTools extends ToolSet> {
         maxRetries: args.maxRetries ?? this.options.maxRetries,
         model: model ?? this.options.chat,
         system: args.system ?? this.options.instructions,
-        messages: [
-          ...contextMessages.map((m) => deserializeMessage(m.message!)),
-          ...messages,
-        ],
+        messages: processedMessages,
       } as T & { model: LanguageModelV1 },
       userId,
       messageId,
@@ -1608,6 +1619,85 @@ export class Agent<AgentTools extends ToolSet> {
     return { embeddings: result.embeddings };
   }
 
+  /**
+   * Process messages to inline file and image URLs that point to localhost
+   * by converting them to base64. This solves the problem of LLMs not being
+   * able to access localhost URLs.
+   */
+  private async _inlineMessagesFiles(
+    messages: CoreMessage[]
+  ): Promise<CoreMessage[]> {
+    // Process each message to convert localhost URLs to base64
+    return Promise.all(
+      messages.map(async (message): Promise<CoreMessage> => {
+        if (
+          message.role !== "user" ||
+          typeof message.content === "string" ||
+          !Array.isArray(message.content)
+        ) {
+          return message;
+        }
+
+        const processedContent = await Promise.all(
+          message.content.map(async (part) => {
+            if (part.type === "image" && part.image instanceof URL) {
+              if (this._isLocalhostUrl(part.image)) {
+                const imageData = await this._downloadFile(part.image);
+                return {
+                  ...part,
+                  image: imageData,
+                } as ImagePart;
+              }
+            }
+
+            // Handle file parts
+            if (part.type === "file" && part.data instanceof URL) {
+              if (this._isLocalhostUrl(part.data)) {
+                const fileData = await this._downloadFile(part.data);
+                return {
+                  ...part,
+                  data: fileData,
+                } as FilePart;
+              }
+            }
+
+            return part;
+          })
+        );
+
+        return {
+          ...message,
+          content: processedContent,
+        };
+      })
+    );
+  }
+
+  /**
+   * Check if a URL points to localhost
+   */
+  private _isLocalhostUrl(url: URL): boolean {
+    return (
+      url.hostname === "localhost" ||
+      url.hostname === "127.0.0.1" ||
+      url.hostname === "::1" ||
+      url.hostname === "0.0.0.0"
+    );
+  }
+
+  /**
+   * Download a file from a URL
+   */
+  private async _downloadFile(url: URL): Promise<ArrayBuffer> {
+    // Fetch the file
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+    }
+
+    return await response.arrayBuffer();
+  }
+
   /**
    * WORKFLOW UTILITIES
    */

package/src/client/setup.test.ts

@@ -3,13 +3,13 @@ import { test } from "vitest";
 import { convexTest } from "convex-test";
 export const modules = import.meta.glob("./**/*.*s");
 
-import componentSchema from "../component/schema";
+import componentSchema from "../component/schema.js";
 import {
   defineSchema,
   type GenericSchema,
   type SchemaDefinition,
 } from "convex/server";
-import { type AgentComponent } from ".";
+import { type AgentComponent } from "./index.js";
 import { componentsGeneric } from "convex/server";
 export { componentSchema };
 export const componentModules = import.meta.glob("../component/**/*.ts");

package/src/component/_generated/api.d.ts

@@ -1177,6 +1177,241 @@ export type Mounts = {
         >;
       }>
     >;
+    updateMessage: FunctionReference<
+      "mutation",
+      "public",
+      {
+        messageId: string;
+        patch: {
+          error?: string;
+          message?:
+            | {
+                content:
+                  | string
+                  | Array<
+                      | {
+                          providerOptions?: Record<string, Record<string, any>>;
+                          text: string;
+                          type: "text";
+                        }
+                      | {
+                          image: string | ArrayBuffer;
+                          mimeType?: string;
+                          providerOptions?: Record<string, Record<string, any>>;
+                          type: "image";
+                        }
+                      | {
+                          data: string | ArrayBuffer;
+                          filename?: string;
+                          mimeType: string;
+                          providerOptions?: Record<string, Record<string, any>>;
+                          type: "file";
+                        }
+                    >;
+                providerOptions?: Record<string, Record<string, any>>;
+                role: "user";
+              }
+            | {
+                content:
+                  | string
+                  | Array<
+                      | {
+                          providerOptions?: Record<string, Record<string, any>>;
+                          text: string;
+                          type: "text";
+                        }
+                      | {
+                          data: string | ArrayBuffer;
+                          filename?: string;
+                          mimeType: string;
+                          providerOptions?: Record<string, Record<string, any>>;
+                          type: "file";
+                        }
+                      | {
+                          providerOptions?: Record<string, Record<string, any>>;
+                          signature?: string;
+                          text: string;
+                          type: "reasoning";
+                        }
+                      | {
+                          data: string;
+                          providerOptions?: Record<string, Record<string, any>>;
+                          type: "redacted-reasoning";
+                        }
+                      | {
+                          args: any;
+                          providerOptions?: Record<string, Record<string, any>>;
+                          toolCallId: string;
+                          toolName: string;
+                          type: "tool-call";
+                        }
+                    >;
+                providerOptions?: Record<string, Record<string, any>>;
+                role: "assistant";
+              }
+            | {
+                content: Array<{
+                  args?: any;
+                  experimental_content?: Array<
+                    | { text: string; type: "text" }
+                    | { data: string; mimeType?: string; type: "image" }
+                  >;
+                  isError?: boolean;
+                  providerOptions?: Record<string, Record<string, any>>;
+                  result: any;
+                  toolCallId: string;
+                  toolName: string;
+                  type: "tool-result";
+                }>;
+                providerOptions?: Record<string, Record<string, any>>;
+                role: "tool";
+              }
+            | {
+                content: string;
+                providerOptions?: Record<string, Record<string, any>>;
+                role: "system";
+              };
+          status?: "pending" | "success" | "failed";
+        };
+      },
+      {
+        _creationTime: number;
+        _id: string;
+        agentName?: string;
+        embeddingId?: string;
+        error?: string;
+        fileIds?: Array<string>;
+        finishReason?:
+          | "stop"
+          | "length"
+          | "content-filter"
+          | "tool-calls"
+          | "error"
+          | "other"
+          | "unknown";
+        id?: string;
+        message?:
+          | {
+              content:
+                | string
+                | Array<
+                    | {
+                        providerOptions?: Record<string, Record<string, any>>;
+                        text: string;
+                        type: "text";
+                      }
+                    | {
+                        image: string | ArrayBuffer;
+                        mimeType?: string;
+                        providerOptions?: Record<string, Record<string, any>>;
+                        type: "image";
+                      }
+                    | {
+                        data: string | ArrayBuffer;
+                        filename?: string;
+                        mimeType: string;
+                        providerOptions?: Record<string, Record<string, any>>;
+                        type: "file";
+                      }
+                  >;
+              providerOptions?: Record<string, Record<string, any>>;
+              role: "user";
+            }
+          | {
+              content:
+                | string
+                | Array<
+                    | {
+                        providerOptions?: Record<string, Record<string, any>>;
+                        text: string;
+                        type: "text";
+                      }
+                    | {
+                        data: string | ArrayBuffer;
+                        filename?: string;
+                        mimeType: string;
+                        providerOptions?: Record<string, Record<string, any>>;
+                        type: "file";
+                      }
+                    | {
+                        providerOptions?: Record<string, Record<string, any>>;
+                        signature?: string;
+                        text: string;
+                        type: "reasoning";
+                      }
+                    | {
+                        data: string;
+                        providerOptions?: Record<string, Record<string, any>>;
+                        type: "redacted-reasoning";
+                      }
+                    | {
+                        args: any;
+                        providerOptions?: Record<string, Record<string, any>>;
+                        toolCallId: string;
+                        toolName: string;
+                        type: "tool-call";
+                      }
+                  >;
+              providerOptions?: Record<string, Record<string, any>>;
+              role: "assistant";
+            }
+          | {
+              content: Array<{
+                args?: any;
+                experimental_content?: Array<
+                  | { text: string; type: "text" }
+                  | { data: string; mimeType?: string; type: "image" }
+                >;
+                isError?: boolean;
+                providerOptions?: Record<string, Record<string, any>>;
+                result: any;
+                toolCallId: string;
+                toolName: string;
+                type: "tool-result";
+              }>;
+              providerOptions?: Record<string, Record<string, any>>;
+              role: "tool";
+            }
+          | {
+              content: string;
+              providerOptions?: Record<string, Record<string, any>>;
+              role: "system";
+            };
+        model?: string;
+        order: number;
+        provider?: string;
+        providerMetadata?: Record<string, Record<string, any>>;
+        providerOptions?: Record<string, Record<string, any>>;
+        reasoning?: string;
+        reasoningDetails?: Array<
+          | { signature?: string; text: string; type: "text" }
+          | { data: string; type: "redacted" }
+        >;
+        sources?: Array<{
+          id: string;
+          providerOptions?: Record<string, Record<string, any>>;
+          sourceType: "url";
+          title?: string;
+          url: string;
+        }>;
+        status: "pending" | "success" | "failed";
+        stepOrder: number;
+        text?: string;
+        threadId: string;
+        tool: boolean;
+        usage?: {
+          completionTokens: number;
+          promptTokens: number;
+          totalTokens: number;
+        };
+        userId?: string;
+        warnings?: Array<
+          | { details?: string; setting: string; type: "unsupported-setting" }
+          | { details?: string; tool: any; type: "unsupported-tool" }
+          | { message: string; type: "other" }
+        >;
+      }
+    >;
   };
   streams: {
     addDelta: FunctionReference<

package/src/component/messages.test.ts

@@ -141,4 +141,144 @@ describe("agent", () => {
       stepOrder: 1,
     });
   });
+
+  test("updateMessage updates message content", async () => {
+    const t = convexTest(schema, modules);
+    const thread = await t.mutation(api.threads.createThread, {
+      userId: "test",
+    });
+    const { messages } = await t.mutation(api.messages.addMessages, {
+      threadId: thread._id as Id<"threads">,
+      messages: [{ message: { role: "user", content: "hello" } }],
+    });
+    const messageId = messages[0]._id as Id<"messages">;
+    
+    const updatedMessage = await t.mutation(api.messages.updateMessage, {
+      messageId,
+      patch: {
+        message: { role: "user", content: "updated content" },
+      },
+    });
+    
+    expect(updatedMessage.message).toEqual({
+      role: "user",
+      content: "updated content",
+    });
+  });
+
+  test("updateMessage updates message status", async () => {
+    const t = convexTest(schema, modules);
+    const thread = await t.mutation(api.threads.createThread, {
+      userId: "test",
+    });
+    const { messages } = await t.mutation(api.messages.addMessages, {
+      threadId: thread._id as Id<"threads">,
+      messages: [{ message: { role: "assistant", content: "hello" } }],
+      pending: true,
+    });
+    const messageId = messages[0]._id as Id<"messages">;
+    
+    // Initial status should be pending
+    expect(messages[0].status).toBe("pending");
+    
+    // Update to success
+    const updatedMessage = await t.mutation(api.messages.updateMessage, {
+      messageId,
+      patch: {
+        status: "success",
+      },
+    });
+    
+    expect(updatedMessage.status).toBe("success");
+  });
+
+  test("updateMessage updates error field", async () => {
+    const t = convexTest(schema, modules);
+    const thread = await t.mutation(api.threads.createThread, {
+      userId: "test",
+    });
+    const { messages } = await t.mutation(api.messages.addMessages, {
+      threadId: thread._id as Id<"threads">,
+      messages: [{ message: { role: "assistant", content: "hello" } }],
+      pending: true,
+    });
+    const messageId = messages[0]._id as Id<"messages">;
+    
+    const updatedMessage = await t.mutation(api.messages.updateMessage, {
+      messageId,
+      patch: {
+        status: "failed",
+        error: "Something went wrong",
+      },
+    });
+    
+    expect(updatedMessage.status).toBe("failed");
+    expect(updatedMessage.error).toBe("Something went wrong");
+  });
+
+  test("updateMessage correctly updates tool messages", async () => {
+    const t = convexTest(schema, modules);
+    const thread = await t.mutation(api.threads.createThread, {
+      userId: "test",
+    });
+    const { messages } = await t.mutation(api.messages.addMessages, {
+      threadId: thread._id as Id<"threads">,
+      messages: [{
+        message: {
+          role: "assistant",
+          content: [
+            {
+              type: "tool-call",
+              args: { a: 1 },
+              toolCallId: "1",
+              toolName: "tool",
+            },
+          ],
+        },
+      }],
+    });
+    const messageId = messages[0]._id as Id<"messages">;
+    
+    const updatedMessage = await t.mutation(api.messages.updateMessage, {
+      messageId,
+      patch: {
+        message: {
+          role: "assistant",
+          content: [
+            {
+              type: "tool-call",
+              args: { a: 2, b: 3 },
+              toolCallId: "1",
+              toolName: "tool",
+            },
+          ],
+        },
+      },
+    });
+    
+    expect(updatedMessage.message).toEqual({
+      role: "assistant",
+      content: [
+        {
+          type: "tool-call",
+          args: { a: 2, b: 3 },
+          toolCallId: "1",
+          toolName: "tool",
+        },
+      ],
+    });
+  });
+
+  test("updateMessage throws error for non-existent message", async () => {
+    const t = convexTest(schema, modules);
+    
+    await expect(
+      t.mutation(api.messages.updateMessage, {
+        messageId: "invalidId" as Id<"messages">,
+        patch: {
+          message: { role: "user", content: "test" },
+        },
+      })
+    ).rejects.toThrow();
+  });
 });

package/src/component/messages.ts

@@ -265,6 +265,36 @@ export const commitMessage = mutation({
   returns: v.null(),
   handler: commitMessageHandler,
 });
+
+export const updateMessage = mutation({
+  args: {
+    messageId: v.id("messages"),
+    patch: v.object({
+      message: v.optional(vMessageDoc.fields.message),
+      status: v.optional(vMessageStatus),
+      error: v.optional(v.string()),
+    }),
+  },
+  returns: vMessageDoc,
+  handler: async (ctx, args) => {
+    const message = await ctx.db.get(args.messageId);
+    assert(message, `Message ${args.messageId} not found`);
+
+    const patch: Partial<Doc<"messages">> = {
+      ...args.patch,
+    };
+
+    if (args.patch.message !== undefined) {
+      patch.message = args.patch.message;
+      patch.tool = isTool(args.patch.message);
+      patch.text = extractText(args.patch.message);
+    }
+
+    await ctx.db.patch(args.messageId, patch);
+    return publicMessage((await ctx.db.get(args.messageId))!);
+  },
+});
+
 async function commitMessageHandler(
   ctx: MutationCtx,
   { messageId }: { messageId: Id<"messages"> }

package/src/component/vector/index.ts

@@ -172,6 +172,10 @@ export const updateBatch = mutation({
         model: v.string(),
         id: vVectorId,
         vector: v.array(v.number()),
+        // TODO: support changing the vector length by
+        // deleting from one table and inserting into another.
+        // However this requires updating all the messages that reference
+        // the vector.
       })
     ),
   },

package/src/react/index.ts

@@ -2,11 +2,10 @@
 import type { ErrorMessage } from "convex-helpers";
 import {
   type PaginatedQueryArgs,
-  // usePaginatedQuery,
   type UsePaginatedQueryResult,
   useQuery,
 } from "convex/react";
-import { usePaginatedQuery } from "./usePaginatedQuery.js";
+import { usePaginatedQuery } from "convex-helpers/react";
 import type { FunctionArgs } from "convex/server";
 import { useMemo, useState } from "react";
 import type { MessageDoc } from "../client/index.js";