@convex-dev/agent 0.7.0 → 0.7.2

package/src/vercel/fileMaterialization.ts

@@ -0,0 +1,174 @@
+import type { UIMessageChunk } from "ai";
+import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
+import type { ActionCtx, AgentComponent, MutationCtx } from "./client/types.js";
+
+export type MaterializedFileRef = { url: string; fileId: string };
+
+/**
+ * Replaces oversized inline files in persisted UI chunks with storage URLs.
+ * Recovery uses the accompanying URL-to-file ownership references to attach
+ * the stored files to the durable messages it creates.
+ */
+export async function materializeUIMessageChunkFiles(
+  ctx: ActionCtx,
+  component: AgentComponent,
+  parts: readonly UIMessageChunk[],
+): Promise<{ parts: UIMessageChunk[]; fileRefs: MaterializedFileRef[] }> {
+  const fileRefs: MaterializedFileRef[] = [];
+  const materialized = await Promise.all(
+    parts.map(async (part): Promise<UIMessageChunk> => {
+      if (part.type === "tool-output-available") {
+        const result = await materializeCanonicalToolResultContentFiles(
+          ctx,
+          component,
+          part.output,
+        );
+        fileRefs.push(...result.fileRefs);
+        return { ...part, output: result.output };
+      }
+      if (part.type !== "file" && part.type !== "reasoning-file") {
+        return { ...part };
+      }
+      const file = await materializeInlineFile(
+        ctx,
+        component,
+        part.url,
+        part.mediaType,
+      );
+      if (!file) {
+        return { ...part };
+      }
+      fileRefs.push({ url: file.url, fileId: file.fileId });
+      return { ...part, url: file.url };
+    }),
+  );
+  return { parts: materialized, fileRefs };
+}
+
+/**
+ * Materializes files in the AI SDK's canonical tool-result content output.
+ * Other tool outputs are application-defined and intentionally remain opaque.
+ */
+export async function materializeCanonicalToolResultContentFiles(
+  ctx: ActionCtx | MutationCtx,
+  component: AgentComponent,
+  output: unknown,
+): Promise<{ output: unknown; fileRefs: MaterializedFileRef[] }> {
+  if (!isCanonicalToolResultContent(output)) {
+    return { output, fileRefs: [] };
+  }
+  const fileRefs: MaterializedFileRef[] = [];
+  const value = await Promise.all(
+    output.value.map(async (part): Promise<unknown> => {
+      if (!isCanonicalToolResultFile(part)) return part;
+      const file = await materializeInlineFile(
+        ctx,
+        component,
+        part.data.type === "url"
+          ? part.data.url
+          : part.data.type === "data"
+            ? part.data.data
+            : new TextEncoder().encode(part.data.text),
+        part.mediaType,
+        part.filename,
+      );
+      if (!file) return part;
+      fileRefs.push({ url: file.url, fileId: file.fileId });
+      return {
+        ...part,
+        data: { type: "url", url: file.url },
+      };
+    }),
+  );
+  return { output: { ...output, value }, fileRefs };
+}
+
+async function materializeInlineFile(
+  ctx: ActionCtx | MutationCtx,
+  component: AgentComponent,
+  data: unknown,
+  mediaType: string,
+  filename?: string,
+) {
+  const bytes = decodeInlineFileData(data);
+  if (!bytes) return undefined;
+  if (bytes.byteLength <= MAX_FILE_SIZE) return undefined;
+  const blobBytes = bytes.buffer.slice(
+    bytes.byteOffset,
+    bytes.byteOffset + bytes.byteLength,
+  ) as ArrayBuffer;
+  const { file } = await storeFile(
+    ctx,
+    component,
+    new Blob([blobBytes], {
+      type: mediaType || "application/octet-stream",
+    }),
+    filename ? { filename } : {},
+  );
+  return file;
+}
+
+function isCanonicalToolResultContent(
+  value: unknown,
+): value is { type: "content"; value: unknown[] } {
+  return (
+    isRecord(value) && value.type === "content" && Array.isArray(value.value)
+  );
+}
+
+function isCanonicalToolResultFile(value: unknown): value is {
+  type: "file";
+  data:
+    | { type: "url"; url: string }
+    | { type: "data"; data: unknown }
+    | { type: "text"; text: string };
+  mediaType: string;
+  filename?: string;
+} {
+  return (
+    isRecord(value) &&
+    value.type === "file" &&
+    typeof value.mediaType === "string" &&
+    isRecord(value.data) &&
+    ((value.data.type === "url" && typeof value.data.url === "string") ||
+      value.data.type === "data" ||
+      (value.data.type === "text" && typeof value.data.text === "string"))
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function decodeInlineFileData(value: unknown): Uint8Array | undefined {
+  if (typeof value === "string") {
+    if (!value.startsWith("data:")) {
+      // Remote and already-stored files are references, not inline payloads.
+      if (/^[a-z][a-z0-9+.-]*:/i.test(value)) return undefined;
+      return decodeBase64(value);
+    }
+    const separator = value.indexOf(",");
+    if (separator === -1) return undefined;
+    const metadata = value.slice(5, separator);
+    const payload = value.slice(separator + 1);
+    try {
+      return metadata.includes(";base64")
+        ? decodeBase64(payload)
+        : new TextEncoder().encode(decodeURIComponent(payload));
+    } catch {
+      return undefined;
+    }
+  }
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  return undefined;
+}
+
+function decodeBase64(value: string): Uint8Array | undefined {
+  try {
+    const binary = atob(value);
+    return Uint8Array.from(binary, (character) => character.charCodeAt(0));
+  } catch {
+    return undefined;
+  }
+}

package/src/vercel/index.ts

@@ -818,6 +818,7 @@ export class Agent<
       metadata: args.metadata ? [args.metadata] : undefined,
       skipEmbeddings: args.skipEmbeddings,
       promptMessageId: args.promptMessageId,
+      order: args.order,
       pendingMessageId: args.pendingMessageId,
     });
     const message = messages.at(-1)!;
@@ -1765,6 +1766,7 @@ export class Agent<
         threadId: v.string(),
         userId: v.optional(v.string()),
         promptMessageId: v.optional(v.string()),
+        order: v.optional(v.union(v.number(), v.literal("next"))),
         messages: v.array(vMessageWithMetadata),
         failPendingSteps: v.optional(v.boolean()),
         embeddings: v.optional(vMessageEmbeddings),

package/src/vercel/mapping.test.ts

@@ -465,6 +465,123 @@ describe("mapping", () => {
     );
   });
 
+  test("materializes oversized canonical tool-result files", async () => {
+    const bytes = new Uint8Array(1024 * 65).fill(1);
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async () => ({
+        fileId: "file-123",
+        storageId: "storage-123",
+      }),
+      storage: {
+        getUrl: async () => "https://example.com/file",
+      },
+    } as unknown as ActionCtx;
+    const result = {
+      type: "tool-result" as const,
+      toolCallId: "call-1",
+      toolName: "render",
+      output: {
+        type: "content" as const,
+        value: [
+          {
+            type: "file" as const,
+            data: { type: "data" as const, data: bytes },
+            mediaType: "application/octet-stream",
+            filename: "result.bin",
+          },
+        ],
+      },
+    } satisfies ToolResultPart;
+
+    const { content, fileIds } = await serializeContent(
+      ctx,
+      api as unknown as AgentComponent,
+      [result],
+    );
+
+    expect(fileIds).toEqual(["file-123"]);
+    expect(content).toMatchObject([
+      {
+        type: "tool-result",
+        output: {
+          type: "content",
+          value: [
+            {
+              type: "file",
+              data: { type: "url", url: "https://example.com/file" },
+            },
+          ],
+        },
+      },
+    ]);
+
+    const opaque = { attachment: "data:application/octet-stream,opaque" };
+    await expect(
+      serializeContent(ctx, api as unknown as AgentComponent, [
+        {
+          type: "tool-result",
+          toolCallId: "call-2",
+          toolName: "render",
+          result: opaque,
+        } satisfies Infer<typeof vToolResultPart>,
+      ]),
+    ).resolves.toMatchObject({
+      content: [{ output: { type: "json", value: opaque } }],
+    });
+  });
+
+  test("materializes oversized canonical tool-result text files", async () => {
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async () => ({
+        fileId: "file-123",
+        storageId: "storage-123",
+      }),
+      storage: {
+        getUrl: async () => "https://example.com/file",
+      },
+    } as unknown as ActionCtx;
+    const result = {
+      type: "tool-result" as const,
+      toolCallId: "call-1",
+      toolName: "render",
+      output: {
+        type: "content" as const,
+        value: [
+          {
+            type: "file" as const,
+            data: { type: "text" as const, text: "x".repeat(1024 * 65) },
+            mediaType: "text/plain",
+            filename: "result.txt",
+          },
+        ],
+      },
+    } satisfies ToolResultPart;
+
+    const { content, fileIds } = await serializeContent(
+      ctx,
+      api as unknown as AgentComponent,
+      [result],
+    );
+
+    expect(fileIds).toEqual(["file-123"]);
+    expect(content).toMatchObject([
+      {
+        type: "tool-result",
+        output: {
+          type: "content",
+          value: [
+            {
+              type: "file",
+              data: { type: "url", url: "https://example.com/file" },
+            },
+          ],
+        },
+      },
+    ]);
+  });
+
   test("sanity: fileIds are not returned for small files", async () => {
     const arr = new Uint8Array([1, 2, 3, 4, 5]);
     const ab = arr.buffer.slice(

package/src/vercel/mapping.ts

@@ -44,6 +44,7 @@ import {
 import type { ActionCtx, AgentComponent } from "./client/types.js";
 import type { MutationCtx } from "./client/types.js";
 import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
+import { materializeCanonicalToolResultContentFiles } from "./fileMaterialization.js";
 import type { Infer } from "convex/values";
 import {
   convertUint8ArrayToBase64,
@@ -466,7 +467,20 @@ export async function serializeContent(
           } satisfies Infer<typeof vToolCallPart>;
         }
         case "tool-result": {
-          return serializeToolResult(part, metadata);
+          const output =
+            "output" in part && part.output !== undefined
+              ? part.output
+              : normalizeToolOutput("result" in part ? part.result : undefined);
+          const materialized = await materializeCanonicalToolResultContentFiles(
+            ctx,
+            component,
+            output,
+          );
+          fileIds.push(...materialized.fileRefs.map((ref) => ref.fileId));
+          return serializeToolResult(
+            { ...part, output: materialized.output } as ToolResultPart,
+            metadata,
+          );
         }
         case "reasoning": {
           return {

package/src/vercel/toUIMessages.test.ts

@@ -26,13 +26,19 @@ describe("toUIMessages", () => {
           content: "Hello!",
         },
         text: "Hello!",
+        providerMetadata: { testProvider: { traceId: "trace-123" } },
       }),
     ];
     const uiMessages = toUIMessages(messages);
     expect(uiMessages).toHaveLength(1);
     expect(uiMessages[0].role).toBe("user");
     expect(uiMessages[0].text).toBe("Hello!");
-    expect(uiMessages[0].parts[0]).toEqual({ type: "text", text: "Hello!" });
+    expect(uiMessages[0].parts[0]).toEqual({
+      type: "text",
+      text: "Hello!",
+      state: "done",
+      providerMetadata: { testProvider: { traceId: "trace-123" } },
+    });
   });
 
   it("handles assistant message", () => {