@convex-dev/agent 0.2.3-alpha.0 → 0.2.4-alpha.0

package/src/component/messages.ts

@@ -10,6 +10,7 @@ import {
   DEFAULT_RECENT_MESSAGES,
   extractText,
   isTool,
+  sorted,
 } from "../shared.js";
 import {
   vMessageEmbeddingsWithDimension,
@@ -631,11 +632,11 @@ export const _fetchSearchMessages = internalQuery({
       .map(publicMessage);
     messages.push(...(args.textSearchMessages ?? []));
     // TODO: prioritize more recent messages
-    messages.sort((a, b) => a.order! - b.order!);
+    messages = sorted(messages);
     messages = messages.slice(0, args.limit);
     // Fetch the surrounding messages
     if (!threadId) {
-      return messages.sort((a, b) => a.order - b.order);
+      return messages;
     }
     const included: Record<string, Set<number>> = {};
     for (const m of messages) {
@@ -688,7 +689,7 @@ export const _fetchSearchMessages = internalQuery({
         messages.push(publicMessage(r));
       }
     }
-    return messages.sort((a, b) => a.order - b.order);
+    return sorted(messages);
   },
 });
 

package/src/mapping.ts

@@ -16,19 +16,21 @@ import {
   type TextPart,
   type ToolCallPart,
   type ToolResultPart,
+  type ProviderMetadata,
 } from "ai";
-import type {
-  Message,
-  MessageWithMetadata,
-  Usage,
-  vContent,
-  vFilePart,
-  vImagePart,
-  vReasoningPart,
-  vRedactedReasoningPart,
-  vTextPart,
-  vToolCallPart,
-  vToolResultPart,
+import {
+  vMessageWithMetadata,
+  type Message,
+  type MessageWithMetadata,
+  type Usage,
+  type vContent,
+  type vFilePart,
+  type vImagePart,
+  type vReasoningPart,
+  type vRedactedReasoningPart,
+  type vTextPart,
+  type vToolCallPart,
+  type vToolResultPart,
 } from "./validators.js";
 import type { ActionCtx, AgentComponent } from "./client/types.js";
 import type { RunMutationCtx } from "./client/types.js";
@@ -36,8 +38,10 @@ import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
 import type { Infer } from "convex/values";
 import {
   convertUint8ArrayToBase64,
+  type ProviderOptions,
   type ReasoningPart,
 } from "@ai-sdk/provider-utils";
+import { parse } from "convex-helpers/validators";
 export type AIMessageWithoutId = Omit<AIMessage, "id">;
 
 export type SerializeUrlsAndUint8Arrays<T> = T extends URL
@@ -97,7 +101,9 @@ export async function serializeOrThrow(
   } as SerializedMessage;
 }
 
-export function deserializeMessage(message: SerializedMessage): ModelMessage {
+export function deserializeMessage(
+  message: SerializedMessage | ModelMessage,
+): ModelMessage {
   return {
     ...message,
     content: deserializeContent(message.content),
@@ -171,12 +177,12 @@ export async function serializeNewMessagesInStep<TOOLS extends ToolSet>(
       : step.response.messages.slice(-1)
     ).map(async (msg): Promise<MessageWithMetadata> => {
       const { message, fileIds } = await serializeMessage(ctx, component, msg);
-      return {
+      return parse(vMessageWithMetadata, {
         message,
         ...(message.role === "tool" ? toolFields : assistantFields),
         text: step.text,
         fileIds,
-      };
+      });
     }),
   );
   // TODO: capture step.files separately?
@@ -212,6 +218,16 @@ export async function serializeObjectResult(
   };
 }
 
+function getMimeOrMediaType(part: { mediaType?: string; mimeType?: string }) {
+  if ("mediaType" in part) {
+    return part.mediaType;
+  }
+  if ("mimeType" in part) {
+    return part.mimeType;
+  }
+  return undefined;
+}
+
 export async function serializeContent(
   ctx: ActionCtx | RunMutationCtx,
   component: AgentComponent,
@@ -221,14 +237,15 @@ export async function serializeContent(
     return { content };
   }
   const fileIds: string[] = [];
-  function getMimeType(part: { mediaType?: string; mimeType?: string }) {
-    if ("mediaType" in part) {
-      return part.mediaType;
-    }
-    if ("mimeType" in part) {
-      return part.mimeType;
-    }
-    return undefined;
+  const metadata: {
+    providerOptions?: ProviderOptions;
+    providerMetadata?: ProviderMetadata;
+  } = {};
+  if ("providerOptions" in content) {
+    metadata.providerOptions = content.providerOptions as ProviderOptions;
+  }
+  if ("providerMetadata" in content) {
+    metadata.providerMetadata = content.providerMetadata as ProviderMetadata;
   }
   const serialized = await Promise.all(
     content.map(async (part) => {
@@ -237,7 +254,7 @@ export async function serializeContent(
           return {
             type: part.type,
             text: part.text,
-            providerOptions: part.providerOptions,
+            ...metadata,
           } satisfies Infer<typeof vTextPart>;
         }
         case "image": {
@@ -250,7 +267,7 @@ export async function serializeContent(
               ctx,
               component,
               new Blob([image], {
-                type: getMimeType(part) || guessMimeType(image),
+                type: getMimeOrMediaType(part) || guessMimeType(image),
               }),
             );
             image = file.url;
@@ -258,8 +275,8 @@ export async function serializeContent(
           }
           return {
             type: part.type,
-            mimeType: getMimeType(part),
-            providerOptions: part.providerOptions,
+            mimeType: getMimeOrMediaType(part),
+            ...metadata,
             image,
           } satisfies Infer<typeof vImagePart>;
         }
@@ -269,7 +286,7 @@ export async function serializeContent(
             const { file } = await storeFile(
               ctx,
               component,
-              new Blob([data], { type: getMimeType(part) }),
+              new Blob([data], { type: getMimeOrMediaType(part) }),
             );
             data = file.url;
             fileIds.push(file.fileId);
@@ -278,8 +295,8 @@ export async function serializeContent(
             type: part.type,
             data,
             filename: part.filename,
-            mimeType: getMimeType(part)!,
-            providerOptions: part.providerOptions,
+            mimeType: getMimeOrMediaType(part)!,
+            ...metadata,
           } satisfies Infer<typeof vFilePart>;
         }
         case "tool-call": {
@@ -289,8 +306,8 @@ export async function serializeContent(
             args: args ?? null,
             toolCallId: part.toolCallId,
             toolName: part.toolName,
-            providerOptions: part.providerOptions,
             providerExecuted: part.providerExecuted,
+            ...metadata,
           } satisfies Infer<typeof vToolCallPart>;
         }
         case "tool-result": {
@@ -300,14 +317,14 @@ export async function serializeContent(
             result: result ?? null,
             toolCallId: part.toolCallId,
             toolName: part.toolName,
-            providerOptions: part.providerOptions,
+            ...metadata,
           } satisfies Infer<typeof vToolResultPart>;
         }
         case "reasoning": {
           return {
             type: part.type,
             text: part.text,
-            providerOptions: part.providerOptions,
+            ...metadata,
           } satisfies Infer<typeof vReasoningPart>;
         }
         // Not in current generation output, but could be in historical messages
@@ -315,7 +332,7 @@ export async function serializeContent(
           return {
             type: part.type,
             data: part.data,
-            providerOptions: part.providerOptions,
+            ...metadata,
           } satisfies Infer<typeof vRedactedReasoningPart>;
         }
         default:
@@ -329,62 +346,90 @@ export async function serializeContent(
   };
 }
 
-export function deserializeContent(content: SerializedContent): Content {
+export function deserializeContent(
+  content: SerializedContent | ModelMessage["content"],
+): Content {
   if (typeof content === "string") {
     return content;
   }
   return content.map((part) => {
+    const metadata: {
+      providerOptions?: ProviderOptions;
+      providerMetadata?: ProviderMetadata;
+    } = {};
+    if ("providerOptions" in part) {
+      metadata.providerOptions = part.providerOptions;
+    }
+    if ("providerMetadata" in part) {
+      metadata.providerMetadata = part.providerMetadata;
+    }
     switch (part.type) {
       case "text":
         return {
           type: part.type,
           text: part.text,
-          providerOptions: part.providerOptions,
+          ...metadata,
         } satisfies TextPart;
       case "image":
         return {
           type: part.type,
           image: deserializeUrl(part.image),
-          mediaType: part.mimeType,
-          providerOptions: part.providerOptions,
+          mediaType: getMimeOrMediaType(part),
+          ...metadata,
         } satisfies ImagePart;
       case "file":
         return {
           type: part.type,
           data: deserializeUrl(part.data),
           filename: part.filename,
-          mediaType: part.mimeType,
-          providerOptions: part.providerOptions,
+          mediaType: getMimeOrMediaType(part)!,
+          ...metadata,
         } satisfies FilePart;
-      case "tool-call":
+      case "tool-call": {
+        const input = "input" in part ? part.input : part.args;
         return {
           type: part.type,
-          input: part.args ?? null,
-          providerExecuted: part.providerExecuted,
+          input: input ?? null,
           toolCallId: part.toolCallId,
           toolName: part.toolName,
-          providerOptions: part.providerOptions,
+          providerExecuted: part.providerExecuted,
+          ...metadata,
         } satisfies ToolCallPart;
-      case "tool-result":
+      }
+      case "tool-result": {
+        const result = "output" in part ? part.output : part.result;
         return {
           type: part.type,
-          output: part.result ?? null,
+          output: result ?? null,
           toolCallId: part.toolCallId,
           toolName: part.toolName,
-          providerOptions: part.providerOptions,
+          ...metadata,
         } satisfies ToolResultPart;
+      }
       case "reasoning":
         return {
           type: part.type,
           text: part.text,
-          providerOptions: part.providerOptions,
+          ...metadata,
         } satisfies ReasoningPart;
       case "redacted-reasoning":
         // TODO: should we just drop this?
         return {
           type: "reasoning",
-          text: part.data,
-          providerOptions: part.providerOptions,
+          text: "",
+          ...metadata,
+          providerOptions: metadata.providerOptions
+            ? {
+                ...Object.fromEntries(
+                  Object.entries(metadata.providerOptions ?? {}).map(
+                    ([key, value]) => [
+                      key,
+                      { ...value, redactedData: part.data },
+                    ],
+                  ),
+                ),
+              }
+            : undefined,
         } satisfies ReasoningPart;
       default:
         return part satisfies Content;
@@ -483,8 +528,11 @@ export function serializeDataOrUrl(
 }
 
 export function deserializeUrl(
-  urlOrString: string | ArrayBuffer,
+  urlOrString: string | ArrayBuffer | URL | DataContent,
 ): URL | DataContent {
+  if (urlOrString instanceof URL) {
+    return urlOrString;
+  }
   if (typeof urlOrString === "string") {
     if (
       urlOrString.startsWith("http://") ||