@convex-dev/agent 0.7.0 → 0.7.1

package/src/vercel/client/streaming.ts

@@ -204,6 +204,12 @@ export class DeltaStreamer<T> {
     throttleMs: number;
     onAsyncAbort: (reason: string) => Promise<void>;
     compress: ((parts: T[]) => T[]) | null;
+    materialize:
+      | ((parts: T[]) => Promise<{
+          parts: T[];
+          fileRefs: Array<{ url: string; fileId: string }>;
+        }>)
+      | null;
   };
   #nextParts: T[] = [];
   #latestWrite: number = 0;
@@ -223,6 +229,10 @@ export class DeltaStreamer<T> {
       onAsyncAbort: (reason: string) => Promise<void>;
       abortSignal: AbortSignal | undefined;
       compress: ((parts: T[]) => T[]) | null;
+      materialize?: (parts: T[]) => Promise<{
+        parts: T[];
+        fileRefs: Array<{ url: string; fileId: string }>;
+      }>;
     },
     public readonly metadata: {
       threadId: string;
@@ -240,6 +250,7 @@ export class DeltaStreamer<T> {
       throttleMs: config.throttleMs ?? DEFAULT_STREAMING_OPTIONS.throttleMs,
       onAsyncAbort: config.onAsyncAbort,
       compress: config.compress,
+      materialize: config.materialize ?? null,
     };
     this.#nextParts = [];
     this.abortController = new AbortController();
@@ -300,8 +311,18 @@ export class DeltaStreamer<T> {
   }
 
   public async consumeStream(stream: AsyncIterableStream<T>) {
-    for await (const chunk of stream) {
-      await this.addParts([chunk]);
+    try {
+      for await (const chunk of stream) {
+        await this.addParts([chunk]);
+      }
+    } catch (error) {
+      // A provider can throw while responding to an abort. Join the durable
+      // abort transition here, outside the active delta writer, before
+      // preserving the provider error for the caller.
+      await this.#abort(
+        error instanceof Error ? error.message : "stream consumption failed",
+      ).catch(() => {});
+      throw error;
     }
     // Skip finish if it will be handled externally (atomically with message save)
     // or if the stream was aborted (e.g., due to a failed delta write).
@@ -335,21 +356,19 @@ export class DeltaStreamer<T> {
     if (this.abortController.signal.aborted) {
       return;
     }
-    const delta = this.#createDelta();
-    if (!delta) {
-      return;
-    }
-    this.#latestWrite = Date.now();
     let success: boolean;
     try {
+      const delta = await this.#createDelta();
+      if (!delta) {
+        return;
+      }
+      this.#latestWrite = Date.now();
       success = await this.ctx.runMutation(
         this.component.streams.addDelta,
         delta,
       );
     } catch (e) {
-      await this.#abortDelta(
-        e instanceof Error ? e.message : "unknown error",
-      );
+      await this.#abortDelta(e instanceof Error ? e.message : "unknown error");
       return;
     }
     if (!success) {
@@ -375,21 +394,36 @@ export class DeltaStreamer<T> {
     }
   }
 
-  #createDelta(): StreamDelta | undefined {
+  async #createDelta(): Promise<
+    | (StreamDelta & { fileRefs?: Array<{ url: string; fileId: string }> })
+    | undefined
+  > {
     if (this.#nextParts.length === 0) {
       return undefined;
     }
     const start = this.#cursor;
-    const end = start + this.#nextParts.length;
+    const pendingParts = this.#nextParts;
+    const end = start + pendingParts.length;
     this.#cursor = end;
-    const parts = this.config.compress
-      ? this.config.compress(this.#nextParts)
-      : this.#nextParts;
     this.#nextParts = [];
+    const materialized = this.config.materialize
+      ? await this.config.materialize(pendingParts)
+      : { parts: pendingParts, fileRefs: [] };
+    const parts = this.config.compress
+      ? this.config.compress(materialized.parts)
+      : materialized.parts;
     if (!this.streamId) {
       throw new Error("Creating a delta before the stream is created");
     }
-    return { streamId: this.streamId, start, end, parts };
+    return {
+      streamId: this.streamId,
+      start,
+      end,
+      parts,
+      ...(materialized.fileRefs.length > 0
+        ? { fileRefs: materialized.fileRefs }
+        : {}),
+    };
   }
 
   public async finish() {

package/src/vercel/fileMaterialization.ts

@@ -0,0 +1,174 @@
+import type { UIMessageChunk } from "ai";
+import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
+import type { ActionCtx, AgentComponent, MutationCtx } from "./client/types.js";
+
+export type MaterializedFileRef = { url: string; fileId: string };
+
+/**
+ * Replaces oversized inline files in persisted UI chunks with storage URLs.
+ * Recovery uses the accompanying URL-to-file ownership references to attach
+ * the stored files to the durable messages it creates.
+ */
+export async function materializeUIMessageChunkFiles(
+  ctx: ActionCtx,
+  component: AgentComponent,
+  parts: readonly UIMessageChunk[],
+): Promise<{ parts: UIMessageChunk[]; fileRefs: MaterializedFileRef[] }> {
+  const fileRefs: MaterializedFileRef[] = [];
+  const materialized = await Promise.all(
+    parts.map(async (part): Promise<UIMessageChunk> => {
+      if (part.type === "tool-output-available") {
+        const result = await materializeCanonicalToolResultContentFiles(
+          ctx,
+          component,
+          part.output,
+        );
+        fileRefs.push(...result.fileRefs);
+        return { ...part, output: result.output };
+      }
+      if (part.type !== "file" && part.type !== "reasoning-file") {
+        return { ...part };
+      }
+      const file = await materializeInlineFile(
+        ctx,
+        component,
+        part.url,
+        part.mediaType,
+      );
+      if (!file) {
+        return { ...part };
+      }
+      fileRefs.push({ url: file.url, fileId: file.fileId });
+      return { ...part, url: file.url };
+    }),
+  );
+  return { parts: materialized, fileRefs };
+}
+
+/**
+ * Materializes files in the AI SDK's canonical tool-result content output.
+ * Other tool outputs are application-defined and intentionally remain opaque.
+ */
+export async function materializeCanonicalToolResultContentFiles(
+  ctx: ActionCtx | MutationCtx,
+  component: AgentComponent,
+  output: unknown,
+): Promise<{ output: unknown; fileRefs: MaterializedFileRef[] }> {
+  if (!isCanonicalToolResultContent(output)) {
+    return { output, fileRefs: [] };
+  }
+  const fileRefs: MaterializedFileRef[] = [];
+  const value = await Promise.all(
+    output.value.map(async (part): Promise<unknown> => {
+      if (!isCanonicalToolResultFile(part)) return part;
+      const file = await materializeInlineFile(
+        ctx,
+        component,
+        part.data.type === "url"
+          ? part.data.url
+          : part.data.type === "data"
+            ? part.data.data
+            : new TextEncoder().encode(part.data.text),
+        part.mediaType,
+        part.filename,
+      );
+      if (!file) return part;
+      fileRefs.push({ url: file.url, fileId: file.fileId });
+      return {
+        ...part,
+        data: { type: "url", url: file.url },
+      };
+    }),
+  );
+  return { output: { ...output, value }, fileRefs };
+}
+
+async function materializeInlineFile(
+  ctx: ActionCtx | MutationCtx,
+  component: AgentComponent,
+  data: unknown,
+  mediaType: string,
+  filename?: string,
+) {
+  const bytes = decodeInlineFileData(data);
+  if (!bytes) return undefined;
+  if (bytes.byteLength <= MAX_FILE_SIZE) return undefined;
+  const blobBytes = bytes.buffer.slice(
+    bytes.byteOffset,
+    bytes.byteOffset + bytes.byteLength,
+  ) as ArrayBuffer;
+  const { file } = await storeFile(
+    ctx,
+    component,
+    new Blob([blobBytes], {
+      type: mediaType || "application/octet-stream",
+    }),
+    filename ? { filename } : {},
+  );
+  return file;
+}
+
+function isCanonicalToolResultContent(
+  value: unknown,
+): value is { type: "content"; value: unknown[] } {
+  return (
+    isRecord(value) && value.type === "content" && Array.isArray(value.value)
+  );
+}
+
+function isCanonicalToolResultFile(value: unknown): value is {
+  type: "file";
+  data:
+    | { type: "url"; url: string }
+    | { type: "data"; data: unknown }
+    | { type: "text"; text: string };
+  mediaType: string;
+  filename?: string;
+} {
+  return (
+    isRecord(value) &&
+    value.type === "file" &&
+    typeof value.mediaType === "string" &&
+    isRecord(value.data) &&
+    ((value.data.type === "url" && typeof value.data.url === "string") ||
+      value.data.type === "data" ||
+      (value.data.type === "text" && typeof value.data.text === "string"))
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function decodeInlineFileData(value: unknown): Uint8Array | undefined {
+  if (typeof value === "string") {
+    if (!value.startsWith("data:")) {
+      // Remote and already-stored files are references, not inline payloads.
+      if (/^[a-z][a-z0-9+.-]*:/i.test(value)) return undefined;
+      return decodeBase64(value);
+    }
+    const separator = value.indexOf(",");
+    if (separator === -1) return undefined;
+    const metadata = value.slice(5, separator);
+    const payload = value.slice(separator + 1);
+    try {
+      return metadata.includes(";base64")
+        ? decodeBase64(payload)
+        : new TextEncoder().encode(decodeURIComponent(payload));
+    } catch {
+      return undefined;
+    }
+  }
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  return undefined;
+}
+
+function decodeBase64(value: string): Uint8Array | undefined {
+  try {
+    const binary = atob(value);
+    return Uint8Array.from(binary, (character) => character.charCodeAt(0));
+  } catch {
+    return undefined;
+  }
+}

package/src/vercel/mapping.test.ts

@@ -465,6 +465,123 @@ describe("mapping", () => {
     );
   });
 
+  test("materializes oversized canonical tool-result files", async () => {
+    const bytes = new Uint8Array(1024 * 65).fill(1);
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async () => ({
+        fileId: "file-123",
+        storageId: "storage-123",
+      }),
+      storage: {
+        getUrl: async () => "https://example.com/file",
+      },
+    } as unknown as ActionCtx;
+    const result = {
+      type: "tool-result" as const,
+      toolCallId: "call-1",
+      toolName: "render",
+      output: {
+        type: "content" as const,
+        value: [
+          {
+            type: "file" as const,
+            data: { type: "data" as const, data: bytes },
+            mediaType: "application/octet-stream",
+            filename: "result.bin",
+          },
+        ],
+      },
+    } satisfies ToolResultPart;
+
+    const { content, fileIds } = await serializeContent(
+      ctx,
+      api as unknown as AgentComponent,
+      [result],
+    );
+
+    expect(fileIds).toEqual(["file-123"]);
+    expect(content).toMatchObject([
+      {
+        type: "tool-result",
+        output: {
+          type: "content",
+          value: [
+            {
+              type: "file",
+              data: { type: "url", url: "https://example.com/file" },
+            },
+          ],
+        },
+      },
+    ]);
+
+    const opaque = { attachment: "data:application/octet-stream,opaque" };
+    await expect(
+      serializeContent(ctx, api as unknown as AgentComponent, [
+        {
+          type: "tool-result",
+          toolCallId: "call-2",
+          toolName: "render",
+          result: opaque,
+        } satisfies Infer<typeof vToolResultPart>,
+      ]),
+    ).resolves.toMatchObject({
+      content: [{ output: { type: "json", value: opaque } }],
+    });
+  });
+
+  test("materializes oversized canonical tool-result text files", async () => {
+    const ctx = {
+      runAction: async () => undefined,
+      runMutation: async () => ({
+        fileId: "file-123",
+        storageId: "storage-123",
+      }),
+      storage: {
+        getUrl: async () => "https://example.com/file",
+      },
+    } as unknown as ActionCtx;
+    const result = {
+      type: "tool-result" as const,
+      toolCallId: "call-1",
+      toolName: "render",
+      output: {
+        type: "content" as const,
+        value: [
+          {
+            type: "file" as const,
+            data: { type: "text" as const, text: "x".repeat(1024 * 65) },
+            mediaType: "text/plain",
+            filename: "result.txt",
+          },
+        ],
+      },
+    } satisfies ToolResultPart;
+
+    const { content, fileIds } = await serializeContent(
+      ctx,
+      api as unknown as AgentComponent,
+      [result],
+    );
+
+    expect(fileIds).toEqual(["file-123"]);
+    expect(content).toMatchObject([
+      {
+        type: "tool-result",
+        output: {
+          type: "content",
+          value: [
+            {
+              type: "file",
+              data: { type: "url", url: "https://example.com/file" },
+            },
+          ],
+        },
+      },
+    ]);
+  });
+
   test("sanity: fileIds are not returned for small files", async () => {
     const arr = new Uint8Array([1, 2, 3, 4, 5]);
     const ab = arr.buffer.slice(

package/src/vercel/mapping.ts

@@ -44,6 +44,7 @@ import {
 import type { ActionCtx, AgentComponent } from "./client/types.js";
 import type { MutationCtx } from "./client/types.js";
 import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
+import { materializeCanonicalToolResultContentFiles } from "./fileMaterialization.js";
 import type { Infer } from "convex/values";
 import {
   convertUint8ArrayToBase64,
@@ -466,7 +467,20 @@ export async function serializeContent(
           } satisfies Infer<typeof vToolCallPart>;
         }
         case "tool-result": {
-          return serializeToolResult(part, metadata);
+          const output =
+            "output" in part && part.output !== undefined
+              ? part.output
+              : normalizeToolOutput("result" in part ? part.result : undefined);
+          const materialized = await materializeCanonicalToolResultContentFiles(
+            ctx,
+            component,
+            output,
+          );
+          fileIds.push(...materialized.fileRefs.map((ref) => ref.fileId));
+          return serializeToolResult(
+            { ...part, output: materialized.output } as ToolResultPart,
+            metadata,
+          );
         }
         case "reasoning": {
           return {

package/src/vercel/toUIMessages.test.ts

@@ -26,13 +26,19 @@ describe("toUIMessages", () => {
           content: "Hello!",
         },
         text: "Hello!",
+        providerMetadata: { testProvider: { traceId: "trace-123" } },
       }),
     ];
     const uiMessages = toUIMessages(messages);
     expect(uiMessages).toHaveLength(1);
     expect(uiMessages[0].role).toBe("user");
     expect(uiMessages[0].text).toBe("Hello!");
-    expect(uiMessages[0].parts[0]).toEqual({ type: "text", text: "Hello!" });
+    expect(uiMessages[0].parts[0]).toEqual({
+      type: "text",
+      text: "Hello!",
+      state: "done",
+      providerMetadata: { testProvider: { traceId: "trace-123" } },
+    });
   });
 
   it("handles assistant message", () => {