@convex-dev/agent 0.6.3 → 0.7.0-alpha.0

package/src/validators.ts

@@ -15,6 +15,11 @@ export const vProviderMetadata = vProviderOptions;
 const providerMetadata = providerOptions;
 export type ProviderMetadata = Infer<typeof providerMetadata>;
 
+export const vProviderReference = v.object({
+  type: v.literal("reference"),
+  reference: v.record(v.string(), v.string()),
+});
+
 export const vThreadStatus = v.union(
   v.literal("active"),
   v.literal("archived"), // unused
@@ -42,7 +47,7 @@ export const vTextPart = v.object({
 
 export const vImagePart = v.object({
   type: v.literal("image"),
-  image: v.union(v.string(), v.bytes()),
+  image: v.union(v.string(), v.bytes(), vProviderReference),
   mediaType: v.optional(v.string()),
   /** @deprecated Use `mediaType` instead. */
   mimeType: v.optional(v.string()),
@@ -51,7 +56,7 @@ export const vImagePart = v.object({
 
 export const vFilePart = v.object({
   type: v.literal("file"),
-  data: v.union(v.string(), v.bytes()),
+  data: v.union(v.string(), v.bytes(), vProviderReference),
   filename: v.optional(v.string()),
   mediaType: v.optional(v.string()),
   /** @deprecated Use `mediaType` instead. */
@@ -73,6 +78,21 @@ export const vReasoningPart = v.object({
   providerMetadata,
 });
 
+export const vReasoningFilePart = v.object({
+  type: v.literal("reasoning-file"),
+  url: v.optional(v.string()),
+  data: v.optional(v.union(v.string(), v.bytes())),
+  mediaType: v.string(),
+  providerOptions,
+  providerMetadata,
+});
+
+export const vCustomContentPart = v.object({
+  type: v.literal("custom"),
+  kind: v.string(),
+  providerOptions,
+  providerMetadata,
+});
 export const vRedactedReasoningPart = v.object({
   type: v.literal("redacted-reasoning"),
   data: v.string(),
@@ -127,6 +147,8 @@ export const vToolCallPart = v.union(
     /** @deprecated Use `input` instead. */
     args: v.optional(v.any()),
     providerExecuted: v.optional(v.boolean()),
+    title: v.optional(v.string()),
+    toolMetadata: v.optional(v.record(v.string(), v.any())),
     providerOptions,
     providerMetadata,
   }),
@@ -139,6 +161,8 @@ export const vToolCallPart = v.union(
     args: v.any(),
     input: v.optional(v.any()),
     providerExecuted: v.optional(v.boolean()),
+    title: v.optional(v.string()),
+    toolMetadata: v.optional(v.record(v.string(), v.any())),
     providerOptions,
     providerMetadata,
   }),
@@ -178,6 +202,21 @@ export const vToolResultOutput = v.union(
           text: v.string(),
           providerOptions,
         }),
+        v.object({
+          type: v.literal("file"),
+          data: v.union(
+            v.object({ type: v.literal("url"), url: v.string() }),
+            vProviderReference,
+            v.object({
+              type: v.literal("data"),
+              data: v.union(v.string(), v.bytes()),
+            }),
+            v.object({ type: v.literal("text"), text: v.string() }),
+          ),
+          mediaType: v.string(),
+          filename: v.optional(v.string()),
+          providerOptions,
+        }),
         /** @deprecated Use `image-data` or `file-data` instead. */
         v.object({
           type: v.literal("media"),
@@ -199,6 +238,7 @@ export const vToolResultOutput = v.union(
         v.object({
           type: v.literal("file-url"),
           url: v.string(),
+          mediaType: v.optional(v.string()),
           providerOptions,
         }),
         v.object({
@@ -214,6 +254,11 @@ export const vToolResultOutput = v.union(
           fileId: v.union(v.string(), v.record(v.string(), v.string())),
           providerOptions,
         }),
+        v.object({
+          type: v.literal("file-reference"),
+          providerReference: v.record(v.string(), v.string()),
+          providerOptions,
+        }),
         v.object({
           type: v.literal("image-data"),
           data: v.string(),
@@ -224,6 +269,11 @@ export const vToolResultOutput = v.union(
           mediaType: v.string(),
           providerOptions,
         }),
+        v.object({
+          type: v.literal("image-file-reference"),
+          providerReference: v.record(v.string(), v.string()),
+          providerOptions,
+        }),
         v.object({
           type: v.literal("image-url"),
           url: v.string(),
@@ -271,6 +321,8 @@ export const vToolApprovalRequest = v.object({
    * ID of the tool call that the approval request is for.
    */
   toolCallId: v.string(),
+  isAutomatic: v.optional(v.boolean()),
+  signature: v.optional(v.string()),
   /** @todo Should we continue to include? */
   providerMetadata,
   /** @todo Should we continue to include? */
@@ -334,6 +386,8 @@ export const vAssistantContent = v.union(
       vTextPart,
       vFilePart,
       vReasoningPart,
+      vReasoningFilePart,
+      vCustomContentPart,
       vRedactedReasoningPart,
       vToolCallPart,
       vToolResultPart,
@@ -383,6 +437,8 @@ export type MessageContentParts =
   | Infer<typeof vImagePart>
   | Infer<typeof vFilePart>
   | Infer<typeof vReasoningPart>
+  | Infer<typeof vReasoningFilePart>
+  | Infer<typeof vCustomContentPart>
   | Infer<typeof vRedactedReasoningPart>
   | Infer<typeof vToolCallPart>
   | Infer<typeof vToolResultPart>
@@ -431,15 +487,34 @@ export const vFinishReason = v.union(
 );
 
 export const vUsage = v.object({
-  promptTokens: v.number(),
-  completionTokens: v.number(),
-  totalTokens: v.number(),
+  promptTokens: v.optional(v.number()),
+  completionTokens: v.optional(v.number()),
+  totalTokens: v.optional(v.number()),
   reasoningTokens: v.optional(v.number()),
   cachedInputTokens: v.optional(v.number()),
+  nonCachedInputTokens: v.optional(v.number()),
+  cacheWriteInputTokens: v.optional(v.number()),
+  textOutputTokens: v.optional(v.number()),
+  raw: v.optional(v.record(v.string(), v.any())),
 });
 export type Usage = Infer<typeof vUsage>;
 
 export const vLanguageModelCallWarning = v.union(
+  v.object({
+    type: v.literal("unsupported"),
+    feature: v.string(),
+    details: v.optional(v.string()),
+  }),
+  v.object({
+    type: v.literal("compatibility"),
+    feature: v.string(),
+    details: v.optional(v.string()),
+  }),
+  v.object({
+    type: v.literal("deprecated"),
+    setting: v.string(),
+    message: v.string(),
+  }),
   v.object({
     type: v.literal("unsupported-setting"),
     setting: v.string(),
@@ -515,8 +590,13 @@ export const vStorageOptions = v.object({
   ),
 });
 
-const vPromptFields = {
-  system: v.optional(v.string()),
+// Generated Convex actions keep the established `system` wire field. The
+// Vercel adapter translates it to AI SDK 7's `instructions` before invoking
+// the model, so action payloads never carry both spellings.
+const vActionPromptFields = {
+  system: v.optional(
+    v.union(v.string(), vSystemMessage, v.array(vSystemMessage)),
+  ),
   prompt: v.optional(v.string()),
   messages: v.optional(v.array(vMessage)),
   promptMessageId: v.optional(v.string()),
@@ -543,7 +623,8 @@ const vCommonArgs = {
   storageOptions: v.optional(vStorageOptions),
   providerOptions,
   callSettings: v.optional(vCallSettings),
-  ...vPromptFields,
+  allowSystemInMessages: v.optional(v.boolean()),
+  ...vActionPromptFields,
 };
 
 export const vTextArgs = v.object({

package/src/{UIMessages.combineUIMessages.test.ts → vercel/UIMessages.combineUIMessages.test.ts}

@@ -1,5 +1,8 @@
 import { describe, it, expect } from "vitest";
-import { combineUIMessages, type UIMessage } from "./UIMessages.js";
+import {
+  combineUIMessages,
+  type UIMessage,
+} from "./UIMessages.js";
 
 describe("combineUIMessages", () => {
   it("should preserve all tool calls when combining messages", () => {

package/src/{UIMessages.ts → vercel/UIMessages.ts}

@@ -1,8 +1,10 @@
 import {
   convertToModelMessages,
   type UIMessage as AIUIMessage,
+  type CustomContentUIPart,
   type DeepPartial,
   type DynamicToolUIPart,
+  type ReasoningFileUIPart,
   type ReasoningUIPart,
   type SourceDocumentUIPart,
   type SourceUrlUIPart,
@@ -13,6 +15,10 @@ import {
   type UIDataTypes,
   type UITools,
 } from "ai";
+import {
+  convertUint8ArrayToBase64,
+  type ReasoningFilePart,
+} from "@ai-sdk/provider-utils";
 import type { Infer } from "convex/values";
 import { toModelMessage, fromModelMessage, toUIFilePart } from "./mapping.js";
 import {
@@ -21,14 +27,14 @@ import {
   isTool,
   joinText,
   sorted,
-} from "./shared.js";
+} from "../shared.js";
 import type {
   MessageDoc,
   MessageStatus,
   ProviderOptions,
   SourcePart,
   vSource,
-} from "./validators.js";
+} from "../validators.js";
 import { omit, pick } from "convex-helpers";
 
 export type UIStatus = "streaming" | MessageStatus;
@@ -153,6 +159,7 @@ function fromSourceParts(parts: UIMessage["parts"]): Infer<typeof vSource>[] {
           id: part.sourceId,
           providerMetadata: part.providerMetadata,
           title: part.title,
+          filename: part.filename,
         } satisfies Infer<typeof vSource>;
       }
       return undefined;
@@ -398,7 +405,13 @@ function createAssistantUIMessage<
 
   // Extract approval parts from raw message content for UI rendering
   type ApprovalPart =
-    | { type: "tool-approval-request"; approvalId: string; toolCallId: string }
+    | {
+        type: "tool-approval-request";
+        approvalId: string;
+        toolCallId: string;
+        isAutomatic?: boolean;
+        signature?: string;
+      }
     | {
         type: "tool-approval-response";
         approvalId: string;
@@ -496,6 +509,16 @@ function createAssistantUIMessage<
             ...contentPart,
           } satisfies ReasoningUIPart);
           break;
+        case "reasoning-file":
+          allParts.push(toUIReasoningFilePart(contentPart));
+          break;
+        case "custom":
+          allParts.push({
+            type: "custom",
+            kind: contentPart.kind,
+            providerMetadata: contentPart.providerOptions,
+          } satisfies CustomContentUIPart);
+          break;
         case "file":
         case "image":
           allParts.push(toUIFilePart(contentPart));
@@ -613,6 +636,8 @@ function createAssistantUIMessage<
           const typedPart = contentPart as {
             toolCallId: string;
             approvalId: string;
+            isAutomatic?: boolean;
+            signature?: string;
           };
           const toolCallPart = allParts.find(
             (part) =>
@@ -623,6 +648,12 @@ function createAssistantUIMessage<
             toolCallPart.state = "approval-requested";
             (toolCallPart as ToolUIPart & { approval?: object }).approval = {
               id: typedPart.approvalId,
+              ...(typedPart.isAutomatic !== undefined
+                ? { isAutomatic: typedPart.isAutomatic }
+                : {}),
+              ...(typedPart.signature !== undefined
+                ? { signature: typedPart.signature }
+                : {}),
             };
           } else {
             console.warn(
@@ -647,9 +678,15 @@ function createAssistantUIMessage<
           ) as ToolUIPart | undefined;
 
           if (toolCallPart) {
+            const approval = (
+              toolCallPart as ToolUIPart & {
+                approval?: { isAutomatic?: boolean; signature?: string };
+              }
+            ).approval;
             if (typedPart.approved) {
               toolCallPart.state = "approval-responded";
               (toolCallPart as ToolUIPart & { approval?: object }).approval = {
+                ...approval,
                 id: typedPart.approvalId,
                 approved: true,
                 reason: typedPart.reason,
@@ -657,6 +694,7 @@ function createAssistantUIMessage<
             } else {
               toolCallPart.state = "output-denied";
               (toolCallPart as ToolUIPart & { approval?: object }).approval = {
+                ...approval,
                 id: typedPart.approvalId,
                 approved: false,
                 reason: typedPart.reason,
@@ -710,6 +748,12 @@ function createAssistantUIMessage<
         // update state if not in a final state
         (toolCallPart as ToolUIPart & { approval?: object }).approval = {
           id: approvalPart.approvalId,
+          ...(approvalPart.isAutomatic !== undefined
+            ? { isAutomatic: approvalPart.isAutomatic }
+            : {}),
+          ...(approvalPart.signature !== undefined
+            ? { signature: approvalPart.signature }
+            : {}),
         };
         if (!finalStates.has(toolCallPart.state)) {
           toolCallPart.state = "approval-requested";
@@ -725,7 +769,13 @@ function createAssistantUIMessage<
 
       if (toolCallPart) {
         // Always update approval info, but only update state if not in a final state
+        const approval = (
+          toolCallPart as ToolUIPart & {
+            approval?: { isAutomatic?: boolean; signature?: string };
+          }
+        ).approval;
         (toolCallPart as ToolUIPart & { approval?: object }).approval = {
+          ...approval,
           id: approvalPart.approvalId,
           approved: approvalPart.approved,
           reason: approvalPart.reason,
@@ -744,8 +794,7 @@ function createAssistantUIMessage<
   // Process execution-denied results to update tool call states
   for (const denied of executionDeniedResults) {
     const toolCallPart = allParts.find(
-      (part) =>
-        "toolCallId" in part && part.toolCallId === denied.toolCallId,
+      (part) => "toolCallId" in part && part.toolCallId === denied.toolCallId,
     ) as ToolUIPart | undefined;
 
     if (toolCallPart) {
@@ -778,6 +827,48 @@ function createAssistantUIMessage<
   };
 }
 
+function toUIReasoningFilePart(part: ReasoningFilePart): ReasoningFileUIPart {
+  const data = part.data;
+  return {
+    type: "reasoning-file",
+    mediaType: part.mediaType,
+    url: toUIReasoningFileUrl(data, part.mediaType),
+    providerMetadata: part.providerOptions,
+  };
+}
+
+function toUIReasoningFileUrl(
+  data: ReasoningFilePart["data"],
+  mediaType: string,
+): string {
+  if (isTaggedReasoningFileData(data)) {
+    if (data.type === "url") return data.url.toString();
+    data = data.data;
+  }
+  if (data instanceof URL) return data.toString();
+  if (typeof data === "string") {
+    return data.startsWith("data:") ? data : `data:${mediaType};base64,${data}`;
+  }
+  const bytes =
+    data instanceof ArrayBuffer
+      ? new Uint8Array(data)
+      : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  return `data:${mediaType};base64,${convertUint8ArrayToBase64(bytes)}`;
+}
+
+function isTaggedReasoningFileData(
+  data: ReasoningFilePart["data"],
+): data is Extract<ReasoningFilePart["data"], { type: string }> {
+  return (
+    typeof data === "object" &&
+    data !== null &&
+    !(data instanceof URL) &&
+    !(data instanceof ArrayBuffer) &&
+    !(data instanceof Uint8Array) &&
+    "type" in data
+  );
+}
+
 function toSourcePart(
   part: SourcePart | Infer<typeof vSource>,
 ): SourceUrlUIPart | SourceDocumentUIPart {

package/src/{client → vercel/client}/approval.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "vitest";
-import { Agent, createTool } from "./index.js";
+import { Agent, createTool } from "../index.js";
 import type {
   DataModelFromSchemaDefinition,
   ApiFromModules,
@@ -65,7 +65,8 @@ const renameFileTool = createTool({
     newName: z.string(),
   }),
   needsApproval: () => true,
-  execute: async (_ctx, input) => `Renamed: ${input.oldName} → ${input.newName}`,
+  execute: async (_ctx, input) =>
+    `Renamed: ${input.oldName} → ${input.newName}`,
 });
 
 // --- Agents (separate mock model instances to avoid shared callIndex) ---
@@ -144,6 +145,14 @@ export const testApproveFlow = action({
       threadId: thread.threadId,
       paginationOpts: { cursor: null, numItems: 20 },
     });
+    const toolResults = allMessages.page.flatMap((message) =>
+      Array.isArray(message.message?.content)
+        ? message.message.content.filter(
+            (part) =>
+              part.type === "tool-result" && part.toolCallId === "tc-approve",
+          )
+        : [],
+    );
 
     return {
       approvalId,
@@ -153,8 +162,9 @@ export const testApproveFlow = action({
       secondSavedCount: result2.savedMessages?.length ?? 0,
       totalThreadMessages: allMessages.page.length,
       threadMessageRoles: allMessages.page.map((m) => m.message?.role),
+      toolResults,
       usageCallCount: usageCalls.length,
-      // Verify usage data includes detail fields (AI SDK v6)
+      // Verify usage data includes detail fields.
       lastUsage: usageCalls.at(-1),
     };
   },
@@ -291,7 +301,12 @@ const multiToolAgent = new Agent(components.agent, {
         },
       ],
       // Step 2: after both tools execute, model responds
-      [{ type: "text", text: "Done! Deleted old.txt and renamed a.txt to b.txt." }],
+      [
+        {
+          type: "text",
+          text: "Done! Deleted old.txt and renamed a.txt to b.txt.",
+        },
+      ],
     ],
   }),
   stopWhen: stepCountIs(5),
@@ -377,14 +392,22 @@ export const submitApprovalForMultiToolAgent = mutation({
 });
 
 export const submitApprovalForApprovalAgent = mutation({
-  args: { threadId: v.string(), approvalId: v.string(), reason: v.optional(v.string()) },
+  args: {
+    threadId: v.string(),
+    approvalId: v.string(),
+    reason: v.optional(v.string()),
+  },
   handler: async (ctx, { threadId, approvalId, reason }) => {
     return approvalAgent.approveToolCall(ctx, { threadId, approvalId, reason });
   },
 });
 
 export const submitDenialForDenialAgent = mutation({
-  args: { threadId: v.string(), approvalId: v.string(), reason: v.optional(v.string()) },
+  args: {
+    threadId: v.string(),
+    approvalId: v.string(),
+    reason: v.optional(v.string()),
+  },
   handler: async (ctx, { threadId, approvalId, reason }) => {
     return denialAgent.denyToolCall(ctx, { threadId, approvalId, reason });
   },
@@ -417,8 +440,6 @@ describe("Tool Approval Workflow", () => {
     expect(result.firstSavedCount).toBeGreaterThanOrEqual(2);
     // Second call: tool-result + assistant text
     expect(result.secondSavedCount).toBeGreaterThanOrEqual(1);
-    // Thread should have (ascending): user, assistant(tool-call+approval),
-    // tool(approval-response), tool(tool-result), assistant(text)
     // listMessages returns descending order:
     expect(result.threadMessageRoles).toEqual([
       "assistant", // final text
@@ -427,9 +448,17 @@ describe("Tool Approval Workflow", () => {
       "assistant", // tool-call + approval-request
       "user", // prompt
     ]);
+    expect(result.toolResults).toEqual([
+      {
+        type: "tool-result",
+        toolCallId: "tc-approve",
+        toolName: "deleteFile",
+        output: { type: "text", value: "Deleted: test.txt" },
+      },
+    ]);
     // Usage handler should be called for each generateText call
     expect(result.usageCallCount).toBeGreaterThanOrEqual(2);
-    // Usage data should include AI SDK v6 detail fields
+    // Usage data should include detail fields.
     expect(result.lastUsage).toBeDefined();
     expect(result.lastUsage!.inputTokenDetails).toBeDefined();
     expect(result.lastUsage!.outputTokenDetails).toBeDefined();
@@ -443,9 +472,7 @@ describe("Tool Approval Workflow", () => {
     expect(result.approvalId).toBeDefined();
     expect(result.firstText).toBe("");
     expect(result.secondText).toBe("OK, I won't delete that file.");
-    // Same message ordering as approve flow:
-    // user, assistant(tool-call+approval), tool(denial-response),
-    // tool(execution-denied result), assistant(text)
+    // Same message ordering as approve flow.
     expect(result.threadMessageRoles).toEqual([
       "assistant",
       "tool",
@@ -469,14 +496,11 @@ describe("Tool Approval Workflow", () => {
     expect(result.secondText).toBe(
       "Done! Deleted old.txt and renamed a.txt to b.txt.",
     );
-    // Both approval responses should be merged into one tool message
-    // (write-time merge in respondToToolCallApproval via findApprovalContext)
-    // Thread: user, assistant(2 tool-calls + 2 approvals),
-    //         tool(2 approval-responses merged), tool(2 tool-results), assistant(text)
+    // Executed results are persisted separately from approval responses.
     expect(result.threadMessageRoles).toEqual([
       "assistant", // final text
       "tool", // tool-results
-      "tool", // approval-responses (merged)
+      "tool", // approval-responses
       "assistant", // tool-calls + approval-requests
       "user", // prompt
     ]);
@@ -485,10 +509,15 @@ describe("Tool Approval Workflow", () => {
   test("approve remains valid with an intervening thread message", async () => {
     usageCalls.length = 0;
     const t = initConvexTest(schema);
-    const result = await t.action(testApi.testApproveFlowWithInterveningMessage, {});
+    const result = await t.action(
+      testApi.testApproveFlowWithInterveningMessage,
+      {},
+    );
 
     expect(result.secondText).toBe("Done! I deleted test.txt.");
     expect(result.approvalResponseOrder).toBe(result.approvalRequestOrder);
-    expect(result.interveningOrder).toBeGreaterThan(result.approvalResponseOrder);
+    expect(result.interveningOrder).toBeGreaterThan(
+      result.approvalResponseOrder,
+    );
   });
 });