@assistant-ui/react-google-adk 0.0.7 → 0.0.8

package/src/AdkClient.test.ts

@@ -137,6 +137,37 @@ describe("createAdkStream - proxy mode", () => {
     });
   });
 
+  it("sends file parts as inlineData in proxy mode", async () => {
+    mockFetch.mockResolvedValueOnce(new Response(sseBody(""), { status: 200 }));
+
+    const stream = createAdkStream({ api: "/api/adk" });
+    const messages: AdkMessage[] = [
+      {
+        id: "m1",
+        type: "human",
+        content: [
+          { type: "text", text: "see attached" },
+          {
+            type: "file",
+            mimeType: "application/pdf",
+            data: "JVBERi0xLjQK",
+            filename: "report.pdf",
+          },
+        ],
+      },
+    ];
+    const gen = await stream(messages, makeConfig());
+    for await (const _ of gen) {
+      /* noop */
+    }
+
+    const body = JSON.parse(mockFetch.mock.calls[0]![1]?.body as string);
+    expect(body.parts).toHaveLength(2);
+    expect(body.parts[1]).toEqual({
+      inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQK" },
+    });
+  });
+
   it("sends parts array when multiple messages are provided", async () => {
     mockFetch.mockResolvedValueOnce(new Response(sseBody(""), { status: 200 }));
 
@@ -608,6 +639,74 @@ describe("createAdkStream - content conversion", () => {
     });
   });
 
+  it("converts file content parts to inlineData in direct mode", async () => {
+    mockFetch.mockResolvedValueOnce(new Response(sseBody(""), { status: 200 }));
+
+    const stream = createAdkStream({
+      api: "http://localhost:8000",
+      appName: "app",
+      userId: "u",
+    });
+    const messages: AdkMessage[] = [
+      {
+        id: "m1",
+        type: "human",
+        content: [
+          {
+            type: "file",
+            mimeType: "application/pdf",
+            data: "JVBERi0xLjQK",
+            filename: "report.pdf",
+          },
+        ],
+      },
+    ];
+    const gen = await stream(messages, makeConfig());
+    for await (const _ of gen) {
+      /* noop */
+    }
+
+    const body = JSON.parse(mockFetch.mock.calls[0]![1]?.body as string);
+    expect(body.newMessage.parts[0]).toEqual({
+      inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQK" },
+    });
+  });
+
+  it("converts file_url content parts to fileData with mimeType in direct mode", async () => {
+    mockFetch.mockResolvedValueOnce(new Response(sseBody(""), { status: 200 }));
+
+    const stream = createAdkStream({
+      api: "http://localhost:8000",
+      appName: "app",
+      userId: "u",
+    });
+    const messages: AdkMessage[] = [
+      {
+        id: "m1",
+        type: "human",
+        content: [
+          {
+            type: "file_url",
+            url: "gs://bucket/report.pdf",
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ];
+    const gen = await stream(messages, makeConfig());
+    for await (const _ of gen) {
+      /* noop */
+    }
+
+    const body = JSON.parse(mockFetch.mock.calls[0]![1]?.body as string);
+    expect(body.newMessage.parts[0]).toEqual({
+      fileData: {
+        fileUri: "gs://bucket/report.pdf",
+        mimeType: "application/pdf",
+      },
+    });
+  });
+
   it("converts code content parts to executableCode", async () => {
     mockFetch.mockResolvedValueOnce(new Response(sseBody(""), { status: 200 }));
 

package/src/AdkClient.ts

@@ -1,7 +1,8 @@
+import { contentToParts } from "./contentToParts";
 import type {
   AdkEvent,
+  AdkEventPart,
   AdkMessage,
-  AdkMessageContentPart,
   AdkStreamCallback,
 } from "./types";
 
@@ -119,9 +120,9 @@ async function resolveHeaders(
  */
 function messagesToContent(messages: AdkMessage[]): {
   role: string;
-  parts: Array<Record<string, unknown>>;
+  parts: AdkEventPart[];
 } {
-  const parts: Array<Record<string, unknown>> = [];
+  const parts: AdkEventPart[] = [];
 
   for (const msg of messages) {
     if (msg.type === "human") {
@@ -209,34 +210,6 @@ function messagesToProxyBody(
   return body;
 }
 
-function contentToParts(
-  content: string | AdkMessageContentPart[],
-): Array<Record<string, unknown>> {
-  if (typeof content === "string") return [{ text: content }];
-  return content.map((part) => {
-    switch (part.type) {
-      case "text":
-        return { text: part.text };
-      case "reasoning":
-        return { text: part.text, thought: true };
-      case "image":
-        return { inlineData: { mimeType: part.mimeType, data: part.data } };
-      case "image_url":
-        return { fileData: { fileUri: part.url } };
-      case "code":
-        return {
-          executableCode: { code: part.code, language: part.language },
-        };
-      case "code_result":
-        return {
-          codeExecutionResult: { output: part.output, outcome: part.outcome },
-        };
-      default:
-        return { text: "" };
-    }
-  });
-}
-
 async function* parseSSEResponse(response: Response): AsyncGenerator<AdkEvent> {
   const reader = response.body!.getReader();
   const decoder = new TextDecoder();

package/src/AdkEventAccumulator.test.ts

@@ -1085,14 +1085,21 @@ describe("AdkEventAccumulator - user message handling", () => {
     });
   });
 
-  it("creates human message with fileData content", () => {
+  it("creates human message with image fileData content", () => {
     const acc = new AdkEventAccumulator();
     const msgs = acc.processEvent(
       makeEvent({
         author: "user",
         content: {
           role: "user",
-          parts: [{ fileData: { fileUri: "gs://bucket/image.png" } }],
+          parts: [
+            {
+              fileData: {
+                fileUri: "gs://bucket/image.png",
+                mimeType: "image/png",
+              },
+            },
+          ],
         },
       }),
     );
@@ -1103,6 +1110,122 @@ describe("AdkEventAccumulator - user message handling", () => {
     });
   });
 
+  it("routes non-image inlineData to a file part", () => {
+    const acc = new AdkEventAccumulator();
+    const msgs = acc.processEvent(
+      makeEvent({
+        author: "user",
+        content: {
+          role: "user",
+          parts: [
+            {
+              inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQK" },
+            },
+          ],
+        },
+      }),
+    );
+    expect(msgs).toHaveLength(1);
+    expect(msgs[0]).toMatchObject({
+      type: "human",
+      content: [
+        { type: "file", mimeType: "application/pdf", data: "JVBERi0xLjQK" },
+      ],
+    });
+  });
+
+  it("routes non-image fileData to a file_url part", () => {
+    const acc = new AdkEventAccumulator();
+    const msgs = acc.processEvent(
+      makeEvent({
+        author: "user",
+        content: {
+          role: "user",
+          parts: [
+            {
+              fileData: {
+                fileUri: "gs://bucket/report.pdf",
+                mimeType: "application/pdf",
+              },
+            },
+          ],
+        },
+      }),
+    );
+    expect(msgs).toHaveLength(1);
+    expect(msgs[0]).toMatchObject({
+      type: "human",
+      content: [
+        {
+          type: "file_url",
+          url: "gs://bucket/report.pdf",
+          mimeType: "application/pdf",
+        },
+      ],
+    });
+  });
+
+  it("coerces fileData with image mime to image_url even if it originated as file_url (intentional, image mime is canonical)", () => {
+    const acc = new AdkEventAccumulator();
+    const msgs = acc.processEvent(
+      makeEvent({
+        author: "user",
+        content: {
+          role: "user",
+          parts: [
+            {
+              fileData: {
+                fileUri: "https://example.com/photo.jpg",
+                mimeType: "image/jpeg",
+              },
+            },
+          ],
+        },
+      }),
+    );
+    expect(msgs).toHaveLength(1);
+    expect(msgs[0]).toMatchObject({
+      type: "human",
+      content: [{ type: "image_url", url: "https://example.com/photo.jpg" }],
+    });
+  });
+
+  it("falls back to image_url for fileData with no mimeType (preserves legacy round-trip)", () => {
+    const acc = new AdkEventAccumulator();
+    const msgs = acc.processEvent(
+      makeEvent({
+        author: "user",
+        content: {
+          role: "user",
+          parts: [{ fileData: { fileUri: "gs://bucket/unknown" } }],
+        },
+      }),
+    );
+    expect(msgs).toHaveLength(1);
+    expect(msgs[0]).toMatchObject({
+      type: "human",
+      content: [{ type: "image_url", url: "gs://bucket/unknown" }],
+    });
+  });
+
+  it("routes assistant non-image inlineData to a file part", () => {
+    const acc = new AdkEventAccumulator();
+    const msgs = acc.processEvent(
+      makeEvent({
+        author: "assistant",
+        content: {
+          role: "model",
+          parts: [{ inlineData: { mimeType: "audio/mpeg", data: "QUJDRA==" } }],
+        },
+      }),
+    );
+    expect(msgs).toHaveLength(1);
+    expect(msgs[0]).toMatchObject({
+      type: "ai",
+      content: [{ type: "file", mimeType: "audio/mpeg", data: "QUJDRA==" }],
+    });
+  });
+
   it("tool result events (no author, role:'user') still create tool messages", () => {
     // Regression: the user-author check must not hijack tool events.
     // messageToEvent for `type:'tool'` emits events without `author`,

package/src/AdkEventAccumulator.ts

@@ -80,6 +80,22 @@ const finishReasonToStatus = (
   return { type: "complete", reason: "stop" };
 };
 
+const inlineDataToPart = (
+  mimeType: string,
+  data: string,
+): AdkMessageContentPart =>
+  mimeType.startsWith("image/")
+    ? { type: "image", mimeType, data }
+    : { type: "file", mimeType, data };
+
+const fileDataToPart = (
+  fileUri: string,
+  mimeType: string | undefined,
+): AdkMessageContentPart =>
+  mimeType == null || mimeType.startsWith("image/")
+    ? { type: "image_url", url: fileUri }
+    : { type: "file_url", url: fileUri, mimeType };
+
 // ── Snake_case normalization ──
 
 const normalizeEventPart = (part: AdkEventPart): AdkEventPart => {
@@ -299,16 +315,13 @@ export class AdkEventAccumulator {
         if (part.text != null && !part.thought) {
           humanParts.push({ type: "text", text: part.text });
         } else if (part.inlineData) {
-          humanParts.push({
-            type: "image",
-            mimeType: part.inlineData.mimeType,
-            data: part.inlineData.data,
-          });
+          humanParts.push(
+            inlineDataToPart(part.inlineData.mimeType, part.inlineData.data),
+          );
         } else if (part.fileData) {
-          humanParts.push({
-            type: "image_url",
-            url: part.fileData.fileUri,
-          });
+          humanParts.push(
+            fileDataToPart(part.fileData.fileUri, part.fileData.mimeType),
+          );
         }
       }
       if (humanParts.length > 0) {
@@ -495,24 +508,21 @@ export class AdkEventAccumulator {
       return;
     }
 
-    // Inline data (images etc)
     if (part.inlineData) {
       const msg = this.getOrCreateAiMessage(event);
-      this.appendContent(msg, {
-        type: "image",
-        mimeType: part.inlineData.mimeType,
-        data: part.inlineData.data,
-      });
+      this.appendContent(
+        msg,
+        inlineDataToPart(part.inlineData.mimeType, part.inlineData.data),
+      );
       return;
     }
 
-    // File data (URI reference)
     if (part.fileData) {
       const msg = this.getOrCreateAiMessage(event);
-      this.appendContent(msg, {
-        type: "image_url",
-        url: part.fileData.fileUri,
-      });
+      this.appendContent(
+        msg,
+        fileDataToPart(part.fileData.fileUri, part.fileData.mimeType),
+      );
     }
   }
 

package/src/contentToParts.ts

@@ -0,0 +1,38 @@
+import type { AdkEventPart, AdkMessage, AdkMessageContentPart } from "./types";
+
+export const contentToParts = (
+  content: AdkMessage["content"],
+): AdkEventPart[] => {
+  if (typeof content === "string") return [{ text: content }];
+  return (content as AdkMessageContentPart[]).map((part): AdkEventPart => {
+    switch (part.type) {
+      case "text":
+        return { text: part.text };
+      case "reasoning":
+        return { text: part.text, thought: true };
+      case "image":
+        return { inlineData: { mimeType: part.mimeType, data: part.data } };
+      case "image_url":
+        return { fileData: { fileUri: part.url } };
+      case "file":
+        return { inlineData: { mimeType: part.mimeType, data: part.data } };
+      case "file_url":
+        return {
+          fileData: {
+            fileUri: part.url,
+            ...(part.mimeType != null && { mimeType: part.mimeType }),
+          },
+        };
+      case "code":
+        return {
+          executableCode: { code: part.code, language: part.language },
+        };
+      case "code_result":
+        return {
+          codeExecutionResult: { output: part.output, outcome: part.outcome },
+        };
+      default:
+        return { text: "" };
+    }
+  });
+};

package/src/convertAdkMessages.test.ts

@@ -78,6 +78,59 @@ describe("convertAdkMessage - ai messages", () => {
     });
   });
 
+  it("converts file content parts to file message parts", () => {
+    const msg: AdkMessage = {
+      id: "m1",
+      type: "ai",
+      content: [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: "JVBERi0xLjQK",
+          filename: "report.pdf",
+        },
+      ],
+    };
+    const result = convertAdkMessage(msg);
+    expect(result).toMatchObject({
+      content: [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: "JVBERi0xLjQK",
+          filename: "report.pdf",
+        },
+      ],
+    });
+  });
+
+  it("converts file_url content parts to data parts", () => {
+    const msg: AdkMessage = {
+      id: "m1",
+      type: "ai",
+      content: [
+        {
+          type: "file_url",
+          url: "gs://bucket/report.pdf",
+          mimeType: "application/pdf",
+        },
+      ],
+    };
+    const result = convertAdkMessage(msg);
+    expect(result).toMatchObject({
+      content: [
+        {
+          type: "data",
+          name: "file_url",
+          data: {
+            url: "gs://bucket/report.pdf",
+            mimeType: "application/pdf",
+          },
+        },
+      ],
+    });
+  });
+
   it("converts code content parts to data parts", () => {
     const msg: AdkMessage = {
       id: "m1",

package/src/convertAdkMessages.ts

@@ -8,6 +8,7 @@ type ContentPart =
   | { type: "text"; text: string }
   | { type: "reasoning"; text: string }
   | { type: "image"; image: string }
+  | { type: "file"; data: string; mimeType: string; filename?: string }
   | { type: "data"; name: string; data: unknown };
 
 const contentToParts = (content: AdkMessage["content"]): ContentPart[] => {
@@ -28,6 +29,22 @@ const contentToParts = (content: AdkMessage["content"]): ContentPart[] => {
           };
         case "image_url":
           return { type: "image", image: part.url };
+        case "file":
+          return {
+            type: "file",
+            data: part.data,
+            mimeType: part.mimeType,
+            ...(part.filename != null && { filename: part.filename }),
+          };
+        case "file_url":
+          return {
+            type: "data",
+            name: "file_url",
+            data: {
+              url: part.url,
+              ...(part.mimeType != null && { mimeType: part.mimeType }),
+            },
+          };
         case "code":
           return {
             type: "data",

package/src/types.ts

@@ -109,6 +109,17 @@ export type AdkMessageContentPart =
   | { type: "reasoning"; text: string }
   | { type: "image"; mimeType: string; data: string }
   | { type: "image_url"; url: string }
+  | {
+      type: "file";
+      mimeType: string;
+      data: string;
+      filename?: string | undefined;
+    }
+  | {
+      type: "file_url";
+      url: string;
+      mimeType?: string | undefined;
+    }
   | { type: "code"; code: string; language: string }
   | { type: "code_result"; output: string; outcome: string }
   | { type: "activity"; message: string };

package/src/useAdkMessages.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it } from "vitest";
+import { messageToEvent } from "./useAdkMessages";
+import type { AdkMessage } from "./types";
+
+describe("messageToEvent (contentToParts)", () => {
+  it("serializes a file content part as inlineData", () => {
+    const msg: AdkMessage = {
+      id: "m1",
+      type: "human",
+      content: [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: "JVBERi0xLjQK",
+          filename: "report.pdf",
+        },
+      ],
+    };
+    const event = messageToEvent(msg);
+    expect(event.content?.parts).toEqual([
+      { inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQK" } },
+    ]);
+  });
+
+  it("serializes a file_url content part as fileData with mimeType", () => {
+    const msg: AdkMessage = {
+      id: "m1",
+      type: "human",
+      content: [
+        {
+          type: "file_url",
+          url: "gs://bucket/report.pdf",
+          mimeType: "application/pdf",
+        },
+      ],
+    };
+    const event = messageToEvent(msg);
+    expect(event.content?.parts).toEqual([
+      {
+        fileData: {
+          fileUri: "gs://bucket/report.pdf",
+          mimeType: "application/pdf",
+        },
+      },
+    ]);
+  });
+
+  it("serializes a file_url without mimeType as bare fileData", () => {
+    const msg: AdkMessage = {
+      id: "m1",
+      type: "human",
+      content: [{ type: "file_url", url: "gs://bucket/unknown" }],
+    };
+    const event = messageToEvent(msg);
+    expect(event.content?.parts).toEqual([
+      { fileData: { fileUri: "gs://bucket/unknown" } },
+    ]);
+  });
+
+  it("serializes mixed text + file content as multiple parts", () => {
+    const msg: AdkMessage = {
+      id: "m1",
+      type: "human",
+      content: [
+        { type: "text", text: "see attached" },
+        { type: "file", mimeType: "image/png", data: "AAAA" },
+      ],
+    };
+    const event = messageToEvent(msg);
+    expect(event.content?.parts).toEqual([
+      { text: "see attached" },
+      { inlineData: { mimeType: "image/png", data: "AAAA" } },
+    ]);
+  });
+});

package/src/useAdkMessages.ts

@@ -3,11 +3,10 @@ import { useState, useCallback, useRef, useMemo } from "react";
 import { v4 as uuidv4 } from "uuid";
 import { useAui } from "@assistant-ui/store";
 import { AdkEventAccumulator } from "./AdkEventAccumulator";
+import { contentToParts } from "./contentToParts";
 import type {
   AdkEvent,
-  AdkEventPart,
   AdkMessage,
-  AdkMessageContentPart,
   AdkMessageMetadata,
   AdkSendMessageConfig,
   AdkStreamCallback,
@@ -200,7 +199,8 @@ export const useAdkMessages = ({
   };
 };
 
-const messageToEvent = (msg: AdkMessage): AdkEvent => {
+/** @internal — exported for unit tests. */
+export const messageToEvent = (msg: AdkMessage): AdkEvent => {
   if (msg.type === "human") {
     return {
       id: msg.id ?? uuidv4(),
@@ -246,31 +246,3 @@ const messageToEvent = (msg: AdkMessage): AdkEvent => {
   };
   return result;
 };
-
-const contentToParts = (
-  content: string | AdkMessageContentPart[],
-): AdkEventPart[] => {
-  if (typeof content === "string") return [{ text: content }];
-  return content.map((part) => {
-    switch (part.type) {
-      case "text":
-        return { text: part.text };
-      case "reasoning":
-        return { text: part.text, thought: true };
-      case "image":
-        return { inlineData: { mimeType: part.mimeType, data: part.data } };
-      case "image_url":
-        return { fileData: { fileUri: part.url } };
-      case "code":
-        return {
-          executableCode: { code: part.code, language: part.language },
-        };
-      case "code_result":
-        return {
-          codeExecutionResult: { output: part.output, outcome: part.outcome },
-        };
-      default:
-        return { text: "" };
-    }
-  });
-};