@assistant-ui/react-ag-ui 0.0.35 → 0.0.38

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@assistant-ui/react-ag-ui",
-  "version": "0.0.35",
+  "version": "0.0.38",
   "description": "AG-UI protocol adapter for assistant-ui",
   "keywords": [
     "ag-ui",
@@ -29,9 +29,9 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@ag-ui/client": "^0.0.54",
-    "@assistant-ui/core": "^0.2.10",
-    "assistant-stream": "^0.3.20",
+    "@ag-ui/client": "^0.0.55",
+    "@assistant-ui/core": "^0.2.14",
+    "assistant-stream": "^0.3.21",
     "fast-json-patch": "^3.1.1"
   },
   "peerDependencies": {
@@ -44,11 +44,11 @@
     }
   },
   "devDependencies": {
-    "@types/react": "^19.2.15",
-    "react": "^19.2.6",
-    "vitest": "^4.1.7",
+    "@types/react": "^19.2.17",
+    "react": "^19.2.7",
+    "vitest": "^4.1.8",
     "zod": "^4.4.3",
-    "@assistant-ui/x-buildutils": "0.0.11"
+    "@assistant-ui/x-buildutils": "0.0.12"
   },
   "publishConfig": {
     "access": "public",

package/src/index.ts

@@ -1,5 +1,7 @@
 export { useAgUiRuntime } from "./useAgUiRuntime";
 export type { AgUiAssistantRuntime } from "./useAgUiRuntime";
+export { fromAgUiMessages } from "./runtime/adapter/conversions";
+export type { FromAgUiMessagesOptions } from "./runtime/adapter/conversions";
 export type {
   AgUiInterrupt,
   AgUiInterruptReason,

package/src/runtime/AgUiThreadRuntimeCore.ts

@@ -722,6 +722,7 @@ export class AgUiThreadRuntimeCore {
         unstable_annotations: [],
         unstable_data: [],
         steps: [],
+        isOptimistic: true,
         custom: {},
       },
     };
@@ -742,9 +743,11 @@ export class AgUiThreadRuntimeCore {
       );
       this.messages = this.messages.filter((m) => m.id !== oldId);
     } else {
-      this.messages = this.messages.map((m) =>
-        m.id === oldId ? { ...m, id: newId } : m,
-      );
+      this.messages = this.messages.map((m) => {
+        if (m.id !== oldId) return m;
+        const { isOptimistic: _, ...metadata } = m.metadata;
+        return { ...m, id: newId, metadata } as ThreadMessage;
+      });
     }
 
     const pendingParent = this.assistantHistoryParents.get(oldId);
@@ -869,6 +872,7 @@ export class AgUiThreadRuntimeCore {
       unstable_annotations: annotations,
       unstable_data: data,
       steps,
+      ...(current.isOptimistic ? { isOptimistic: true } : {}),
       ...(incoming.timing ? { timing: incoming.timing } : {}),
       custom: incoming.custom
         ? { ...current.custom, ...incoming.custom }
@@ -911,16 +915,14 @@ export class AgUiThreadRuntimeCore {
 
   private importMessagesSnapshot(rawMessages: readonly unknown[]) {
     try {
-      const normalized = fromAgUiMessages(rawMessages);
+      const normalized = fromAgUiMessages(rawMessages, {
+        showThinking: this.showThinking,
+      });
       const converted: ThreadMessage[] = [];
       for (const message of normalized) {
         try {
           converted.push(
-            fromThreadMessageLike(
-              message as any,
-              generateId(),
-              FALLBACK_USER_STATUS,
-            ),
+            fromThreadMessageLike(message, generateId(), FALLBACK_USER_STATUS),
           );
         } catch (error) {
           this.logger.error?.(

package/src/runtime/adapter/conversions.ts

@@ -1,7 +1,9 @@
 "use client";
 
 import type { InputContent } from "@ag-ui/client";
+import type { ThreadMessageLike as CoreThreadMessageLike } from "@assistant-ui/core";
 import { type Tool, toToolsJSONSchema } from "assistant-stream";
+import type { ReadonlyJSONObject } from "assistant-stream/utils";
 
 export type { InputContent };
 
@@ -46,9 +48,9 @@ export type AgUiMessage =
 type ToolCallPart = {
   type: "tool-call";
   toolCallId?: string;
-  toolName?: string;
+  toolName: string;
   argsText?: string;
-  args?: Record<string, unknown>;
+  args?: ReadonlyJSONObject;
   result?: unknown;
   isError?: boolean;
   unstable_toolMessageId?: string;
@@ -127,6 +129,40 @@ function parseDataUrl(
 
 const httpUrlPattern = /^https?:\/\//i;
 
+type InputContentSource =
+  | { type: "data"; value: string; mimeType: string }
+  | { type: "url"; value: string; mimeType?: string };
+
+type MediaInputType = "image" | "audio" | "video" | "document";
+
+function mediaTypeForMime(mimeType: string | undefined): MediaInputType {
+  if (mimeType?.startsWith("image/")) return "image";
+  if (mimeType?.startsWith("audio/")) return "audio";
+  if (mimeType?.startsWith("video/")) return "video";
+  return "document";
+}
+
+// Build an AG-UI multimodal source from a data URL, raw base64 payload, or an
+// http(s) URL. A `url` source may omit the mime type; a `data` source always
+// resolves one (falling back to application/octet-stream).
+function buildInputSource(
+  value: string,
+  declaredMimeType: string | undefined,
+): InputContentSource {
+  if (httpUrlPattern.test(value)) {
+    return declaredMimeType !== undefined
+      ? { type: "url", value, mimeType: declaredMimeType }
+      : { type: "url", value };
+  }
+  const parsed = parseDataUrl(value);
+  return {
+    type: "data",
+    value: parsed?.data ?? value,
+    mimeType:
+      parsed?.mimeType ?? declaredMimeType ?? "application/octet-stream",
+  };
+}
+
 function toInputContent(
   part: unknown,
   fallbackMimeType: string | undefined,
@@ -143,57 +179,140 @@ function toInputContent(
   if (type === "image") {
     const image = getString(part, "image");
     if (image === undefined) return null;
-    const parsed = parseDataUrl(image);
-    if (parsed) {
-      return {
-        type: "image",
-        source: {
-          type: "data",
-          value: parsed.data,
-          mimeType: parsed.mimeType,
-        },
-      };
-    }
-    return {
-      type: "image",
-      source: {
-        type: "url",
-        value: image,
-        ...(fallbackMimeType !== undefined
-          ? { mimeType: fallbackMimeType }
-          : {}),
-      },
-    };
+    return { type: "image", source: buildInputSource(image, fallbackMimeType) };
   }
 
   if (type === "file") {
     const data = getString(part, "data");
     if (data === undefined) return null;
-    const partMimeType = getString(part, "mimeType");
+    const declaredMimeType = getString(part, "mimeType") || fallbackMimeType;
     const filename = getString(part, "filename");
-    const mimeType =
-      partMimeType || fallbackMimeType || "application/octet-stream";
-
-    if (httpUrlPattern.test(data)) {
-      return {
-        type: "binary",
-        mimeType,
-        url: data,
-        ...(filename !== undefined ? { filename } : {}),
-      };
+    const source = buildInputSource(data, declaredMimeType);
+    const metadata = filename !== undefined ? { filename } : undefined;
+    switch (mediaTypeForMime(source.mimeType)) {
+      case "image":
+        return { type: "image", source, ...(metadata && { metadata }) };
+      case "audio":
+        return { type: "audio", source, ...(metadata && { metadata }) };
+      case "video":
+        return { type: "video", source, ...(metadata && { metadata }) };
+      default:
+        return { type: "document", source, ...(metadata && { metadata }) };
     }
-    const parsed = parseDataUrl(data);
+  }
+
+  return null;
+}
+
+type SnapshotAttachment = NonNullable<
+  CoreThreadMessageLike["attachments"]
+>[number];
+
+const mediaInputTypes = new Set(["image", "audio", "video", "document"]);
+
+// Inverse of buildInputSource.
+function inputSourceToString(
+  value: unknown,
+): { value: string; mimeType?: string } | null {
+  if (!isObject(value)) return null;
+  const sourceValue = getString(value, "value");
+  if (sourceValue === undefined) return null;
+  const mimeType = getString(value, "mimeType");
+  const type = getString(value, "type");
+  if (type === "url") {
+    return { value: sourceValue, ...(mimeType !== undefined && { mimeType }) };
+  }
+  if (type === "data") {
+    const resolvedMimeType = mimeType ?? "application/octet-stream";
     return {
-      type: "binary",
-      mimeType: parsed?.mimeType ?? mimeType,
-      data: parsed?.data ?? data,
-      ...(filename !== undefined ? { filename } : {}),
+      value: `data:${resolvedMimeType};base64,${sourceValue}`,
+      mimeType: resolvedMimeType,
     };
   }
-
   return null;
 }
 
+// Mirrors @ag-ui/client's BackwardCompatibility_0_0_47 middleware.
+function upgradeBinaryInputPart(
+  part: Record<string, unknown>,
+): Record<string, unknown> | null {
+  const mimeType = getString(part, "mimeType");
+  if (mimeType === undefined) return null;
+  const data = getString(part, "data");
+  const url = getString(part, "url");
+  const source = data
+    ? { type: "data", value: data, mimeType }
+    : url
+      ? { type: "url", value: url, mimeType }
+      : null;
+  if (!source) return null;
+  const filename = getString(part, "filename");
+  return {
+    type: mediaTypeForMime(mimeType),
+    source,
+    ...(filename && { metadata: { filename } }),
+  };
+}
+
+function toSnapshotAttachments(content: unknown): SnapshotAttachment[] {
+  if (!Array.isArray(content)) return [];
+
+  const attachments: SnapshotAttachment[] = [];
+  for (const rawPart of content) {
+    if (!isObject(rawPart)) continue;
+    const part =
+      getString(rawPart, "type") === "binary"
+        ? upgradeBinaryInputPart(rawPart)
+        : rawPart;
+    if (!part) continue;
+    const type = getString(part, "type");
+    if (type === undefined || !mediaInputTypes.has(type)) continue;
+    const source = inputSourceToString(part.source);
+    if (!source) continue;
+
+    const filename = isObject(part.metadata)
+      ? getString(part.metadata, "filename")
+      : undefined;
+    const id = attachments.length.toString();
+
+    if (type === "image") {
+      attachments.push({
+        id,
+        type: "image",
+        name: filename ?? "image",
+        ...(source.mimeType !== undefined && { contentType: source.mimeType }),
+        status: { type: "complete" },
+        content: [
+          {
+            type: "image",
+            image: source.value,
+            ...(filename !== undefined && { filename }),
+          },
+        ],
+      });
+      continue;
+    }
+
+    const mimeType = source.mimeType ?? "application/octet-stream";
+    attachments.push({
+      id,
+      type: type === "document" ? "document" : "file",
+      name: filename ?? "file",
+      contentType: mimeType,
+      status: { type: "complete" },
+      content: [
+        {
+          type: "file",
+          data: source.value,
+          mimeType,
+          ...(filename !== undefined && { filename }),
+        },
+      ],
+    });
+  }
+  return attachments;
+}
+
 function buildUserContent(message: ThreadMessageLike): string | InputContent[] {
   // File parts in message.content are intentionally skipped: the canonical
   // binary payload for files always flows through message.attachments.
@@ -260,9 +379,9 @@ function toToolCallPart(value: unknown): ToolCallPart | null {
     typeof argsText === "string" ? parseJSONText(argsText) : undefined;
   const args =
     isObject(parsedArgs) && !Array.isArray(parsedArgs)
-      ? (parsedArgs as Record<string, unknown>)
+      ? (parsedArgs as ReadonlyJSONObject)
       : isObject(value.args) && !Array.isArray(value.args)
-        ? (value.args as Record<string, unknown>)
+        ? (value.args as ReadonlyJSONObject)
         : undefined;
 
   const part: ToolCallPart = {
@@ -322,7 +441,7 @@ function extractAssistantToolCalls(
 
 function toAssistantSnapshotMessage(
   rawMessage: Record<string, unknown>,
-): ThreadMessageLike {
+): CoreThreadMessageLike {
   const text = extractText(rawMessage.content);
   const toolCallParts = extractAssistantToolCalls(rawMessage);
   const assistantContent = [
@@ -341,20 +460,33 @@ function toAssistantSnapshotMessage(
 function toUserOrSystemSnapshotMessage(
   role: "user" | "system",
   rawMessage: Record<string, unknown>,
-): ThreadMessageLike {
+): CoreThreadMessageLike {
   const messageName = getString(rawMessage, "name");
+  const attachments =
+    role === "user" ? toSnapshotAttachments(rawMessage.content) : [];
   return {
     id: getString(rawMessage, "id") ?? generateId(),
     role,
     content: extractText(rawMessage.content),
     ...(messageName !== undefined ? { name: messageName } : {}),
+    ...(attachments.length > 0 ? { attachments } : {}),
   };
 }
 
+export type FromAgUiMessagesOptions = {
+  /**
+   * Whether to convert `reasoning` messages into visible reasoning parts.
+   * Matches the `showThinking` option of `useAgUiRuntime`. Defaults to `true`.
+   */
+  showThinking?: boolean;
+};
+
 export function fromAgUiMessages(
   messages: readonly unknown[],
-): ThreadMessageLike[] {
-  const converted: ThreadMessageLike[] = [];
+  options?: FromAgUiMessagesOptions,
+): CoreThreadMessageLike[] {
+  const showThinking = options?.showThinking ?? true;
+  const converted: CoreThreadMessageLike[] = [];
 
   for (const rawMessage of messages) {
     if (!isObject(rawMessage)) continue;
@@ -454,6 +586,20 @@ export function fromAgUiMessages(
       continue;
     }
 
+    if (role === "reasoning") {
+      // Gate on showThinking so a cold reload matches the live run: the
+      // aggregator never stores reasoning parts when showThinking is false.
+      if (!showThinking) continue;
+      const text = extractText(rawMessage.content);
+      if (text.trim().length === 0) continue;
+      converted.push({
+        id: getString(rawMessage, "id") ?? generateId(),
+        role: "assistant",
+        content: [{ type: "reasoning", text }],
+      });
+      continue;
+    }
+
     if (role === "user" || role === "system") {
       converted.push(toUserOrSystemSnapshotMessage(role, rawMessage));
     }
@@ -478,6 +624,12 @@ function convertAssistantMessage(
     part,
   }));
 
+  // Drop assistant messages with no text or tool calls (e.g. an imported
+  // reasoning-only entry) so they are not re-sent as a blank assistant turn.
+  if (content.length === 0 && toolCalls.length === 0) {
+    return;
+  }
+
   const assistantMessage: AgUiMessage = {
     id: message.id,
     role: "assistant",

package/src/runtime/adapter/run-aggregator.ts

@@ -1,10 +1,11 @@
 "use client";
 
-import type {
-  ChatModelRunResult,
-  MessageTiming,
-  ThreadAssistantMessagePart,
-  ToolCallMessagePart,
+import {
+  isMcpAppUri,
+  type ChatModelRunResult,
+  type MessageTiming,
+  type ThreadAssistantMessagePart,
+  type ToolCallMessagePart,
 } from "@assistant-ui/core";
 import type { AgUiEvent, AgUiInterrupt } from "../types";
 import type { Logger } from "../logger";
@@ -26,8 +27,11 @@ type ToolCallState = {
   isError: boolean | undefined;
   parentMessageId?: string;
   toolMessageId?: string;
+  mcpAppResourceUri?: string;
 };
 
+const MCP_APPS_ACTIVITY_TYPE = "mcp-apps";
+
 export type RunAggregatorOptions = {
   showThinking: boolean;
   logger: Logger;
@@ -58,6 +62,7 @@ export class RunAggregator {
   private activeReasoningKey: string | undefined;
   private reasoningPartCounter = 0;
   private readonly toolCalls = new Map<string, ToolCallState>();
+  private lastResolvedToolCallId: string | undefined;
   private readonly partOrder: (
     | { kind: "text"; key: string }
     | { kind: "reasoning"; key: string }
@@ -85,6 +90,7 @@ export class RunAggregator {
         this.activeReasoningKey = undefined;
         this.reasoningPartCounter = 0;
         this.toolCalls.clear();
+        this.lastResolvedToolCallId = undefined;
         this.partOrder.length = 0;
         this.textPartCounter = 0;
         this.activeTextMessageId = undefined;
@@ -224,6 +230,21 @@ export class RunAggregator {
         this.emit();
         break;
       }
+      case "ACTIVITY_SNAPSHOT": {
+        if (event.activityType !== MCP_APPS_ACTIVITY_TYPE) break;
+        const id = this.lastResolvedToolCallId;
+        const entry = id ? this.toolCalls.get(id) : undefined;
+        const resourceUri = event.content.resourceUri;
+        if (
+          entry &&
+          typeof resourceUri === "string" &&
+          isMcpAppUri(resourceUri)
+        ) {
+          entry.mcpAppResourceUri = resourceUri;
+          this.emit();
+        }
+        break;
+      }
 
       default: {
         this.logger.debug?.("[agui] aggregator ignored event", event);
@@ -372,6 +393,7 @@ export class RunAggregator {
     if (toolMessageId) {
       entry.toolMessageId = toolMessageId;
     }
+    this.lastResolvedToolCallId = id;
   }
 
   private tryParseJSON(value: string): unknown {
@@ -415,6 +437,9 @@ export class RunAggregator {
         argsText: entry.argsText,
         ...(entry.result !== undefined ? { result: entry.result } : {}),
         ...(entry.isError !== undefined ? { isError: entry.isError } : {}),
+        ...(entry.mcpAppResourceUri
+          ? { mcp: { app: { resourceUri: entry.mcpAppResourceUri } } }
+          : {}),
         ...(entry.parentMessageId ? { parentId: entry.parentMessageId } : {}),
         ...(entry.toolMessageId
           ? { unstable_toolMessageId: entry.toolMessageId }

package/src/runtime/adapter/subscriber.ts

@@ -27,6 +27,7 @@ type Subscriber = {
   onToolCallEndEvent?: (payload: { event: unknown }) => void;
   onToolCallChunkEvent?: (payload: { event: unknown }) => void;
   onToolCallResultEvent?: (payload: { event: unknown }) => void;
+  onActivitySnapshotEvent?: (payload: { event: unknown }) => void;
   onStateSnapshotEvent?: (payload: { event: unknown }) => void;
   onStateDeltaEvent?: (payload: { event: unknown }) => void;
   onMessagesSnapshotEvent?: (payload: { event: unknown }) => void;
@@ -122,6 +123,8 @@ export const createAgUiSubscriber = (
       dispatchIfValid(event, "TOOL_CALL_CHUNK"),
     onToolCallResultEvent: ({ event }) =>
       dispatchIfValid(event, "TOOL_CALL_RESULT"),
+    onActivitySnapshotEvent: ({ event }) =>
+      dispatchIfValid(event, "ACTIVITY_SNAPSHOT"),
     onStateSnapshotEvent: ({ event }) =>
       dispatchIfValid(event, "STATE_SNAPSHOT"),
     onStateDeltaEvent: ({ event }) => dispatchIfValid(event, "STATE_DELTA"),

package/src/runtime/event-parser.ts

@@ -235,6 +235,15 @@ export const parseAgUiEvent = (
           ? (payload.messages as any[])
           : [],
       };
+    case "ACTIVITY_SNAPSHOT": {
+      const activityType = getString("activityType");
+      if (!activityType || !isPlainObject(payload.content)) return null;
+      return {
+        type: "ACTIVITY_SNAPSHOT" as const,
+        activityType,
+        content: payload.content,
+      };
+    }
     case "RAW":
       return withOptional(
         { type: "RAW" as const, event: payload.event },

package/src/runtime/types.ts

@@ -121,6 +121,11 @@ export type AgUiEvent =
       content: string;
       role?: "tool";
     }
+  | {
+      type: "ACTIVITY_SNAPSHOT";
+      activityType: string;
+      content: Record<string, unknown>;
+    }
   | { type: "RAW"; event: any; source?: string }
   | { type: "CUSTOM"; name: string; value: any }
   | { type: "STATE_SNAPSHOT"; snapshot: any }