@assistant-ui/react-ag-ui 0.0.35 → 0.0.36

package/src/runtime/adapter/conversions.ts

@@ -1,7 +1,9 @@
 "use client";
 
 import type { InputContent } from "@ag-ui/client";
+import type { ThreadMessageLike as CoreThreadMessageLike } from "@assistant-ui/core";
 import { type Tool, toToolsJSONSchema } from "assistant-stream";
+import type { ReadonlyJSONObject } from "assistant-stream/utils";
 
 export type { InputContent };
 
@@ -46,9 +48,9 @@ export type AgUiMessage =
 type ToolCallPart = {
   type: "tool-call";
   toolCallId?: string;
-  toolName?: string;
+  toolName: string;
   argsText?: string;
-  args?: Record<string, unknown>;
+  args?: ReadonlyJSONObject;
   result?: unknown;
   isError?: boolean;
   unstable_toolMessageId?: string;
@@ -127,6 +129,40 @@ function parseDataUrl(
 
 const httpUrlPattern = /^https?:\/\//i;
 
+type InputContentSource =
+  | { type: "data"; value: string; mimeType: string }
+  | { type: "url"; value: string; mimeType?: string };
+
+type MediaInputType = "image" | "audio" | "video" | "document";
+
+function mediaTypeForMime(mimeType: string | undefined): MediaInputType {
+  if (mimeType?.startsWith("image/")) return "image";
+  if (mimeType?.startsWith("audio/")) return "audio";
+  if (mimeType?.startsWith("video/")) return "video";
+  return "document";
+}
+
+// Build an AG-UI multimodal source from a data URL, raw base64 payload, or an
+// http(s) URL. A `url` source may omit the mime type; a `data` source always
+// resolves one (falling back to application/octet-stream).
+function buildInputSource(
+  value: string,
+  declaredMimeType: string | undefined,
+): InputContentSource {
+  if (httpUrlPattern.test(value)) {
+    return declaredMimeType !== undefined
+      ? { type: "url", value, mimeType: declaredMimeType }
+      : { type: "url", value };
+  }
+  const parsed = parseDataUrl(value);
+  return {
+    type: "data",
+    value: parsed?.data ?? value,
+    mimeType:
+      parsed?.mimeType ?? declaredMimeType ?? "application/octet-stream",
+  };
+}
+
 function toInputContent(
   part: unknown,
   fallbackMimeType: string | undefined,
@@ -143,52 +179,26 @@ function toInputContent(
   if (type === "image") {
     const image = getString(part, "image");
     if (image === undefined) return null;
-    const parsed = parseDataUrl(image);
-    if (parsed) {
-      return {
-        type: "image",
-        source: {
-          type: "data",
-          value: parsed.data,
-          mimeType: parsed.mimeType,
-        },
-      };
-    }
-    return {
-      type: "image",
-      source: {
-        type: "url",
-        value: image,
-        ...(fallbackMimeType !== undefined
-          ? { mimeType: fallbackMimeType }
-          : {}),
-      },
-    };
+    return { type: "image", source: buildInputSource(image, fallbackMimeType) };
   }
 
   if (type === "file") {
     const data = getString(part, "data");
     if (data === undefined) return null;
-    const partMimeType = getString(part, "mimeType");
+    const declaredMimeType = getString(part, "mimeType") || fallbackMimeType;
     const filename = getString(part, "filename");
-    const mimeType =
-      partMimeType || fallbackMimeType || "application/octet-stream";
-
-    if (httpUrlPattern.test(data)) {
-      return {
-        type: "binary",
-        mimeType,
-        url: data,
-        ...(filename !== undefined ? { filename } : {}),
-      };
+    const source = buildInputSource(data, declaredMimeType);
+    const metadata = filename !== undefined ? { filename } : undefined;
+    switch (mediaTypeForMime(source.mimeType)) {
+      case "image":
+        return { type: "image", source, ...(metadata && { metadata }) };
+      case "audio":
+        return { type: "audio", source, ...(metadata && { metadata }) };
+      case "video":
+        return { type: "video", source, ...(metadata && { metadata }) };
+      default:
+        return { type: "document", source, ...(metadata && { metadata }) };
     }
-    const parsed = parseDataUrl(data);
-    return {
-      type: "binary",
-      mimeType: parsed?.mimeType ?? mimeType,
-      data: parsed?.data ?? data,
-      ...(filename !== undefined ? { filename } : {}),
-    };
   }
 
   return null;
@@ -260,9 +270,9 @@ function toToolCallPart(value: unknown): ToolCallPart | null {
     typeof argsText === "string" ? parseJSONText(argsText) : undefined;
   const args =
     isObject(parsedArgs) && !Array.isArray(parsedArgs)
-      ? (parsedArgs as Record<string, unknown>)
+      ? (parsedArgs as ReadonlyJSONObject)
       : isObject(value.args) && !Array.isArray(value.args)
-        ? (value.args as Record<string, unknown>)
+        ? (value.args as ReadonlyJSONObject)
         : undefined;
 
   const part: ToolCallPart = {
@@ -322,7 +332,7 @@ function extractAssistantToolCalls(
 
 function toAssistantSnapshotMessage(
   rawMessage: Record<string, unknown>,
-): ThreadMessageLike {
+): CoreThreadMessageLike {
   const text = extractText(rawMessage.content);
   const toolCallParts = extractAssistantToolCalls(rawMessage);
   const assistantContent = [
@@ -341,7 +351,7 @@ function toAssistantSnapshotMessage(
 function toUserOrSystemSnapshotMessage(
   role: "user" | "system",
   rawMessage: Record<string, unknown>,
-): ThreadMessageLike {
+): CoreThreadMessageLike {
   const messageName = getString(rawMessage, "name");
   return {
     id: getString(rawMessage, "id") ?? generateId(),
@@ -351,10 +361,20 @@ function toUserOrSystemSnapshotMessage(
   };
 }
 
+export type FromAgUiMessagesOptions = {
+  /**
+   * Whether to convert `reasoning` messages into visible reasoning parts.
+   * Matches the `showThinking` option of `useAgUiRuntime`. Defaults to `true`.
+   */
+  showThinking?: boolean;
+};
+
 export function fromAgUiMessages(
   messages: readonly unknown[],
-): ThreadMessageLike[] {
-  const converted: ThreadMessageLike[] = [];
+  options?: FromAgUiMessagesOptions,
+): CoreThreadMessageLike[] {
+  const showThinking = options?.showThinking ?? true;
+  const converted: CoreThreadMessageLike[] = [];
 
   for (const rawMessage of messages) {
     if (!isObject(rawMessage)) continue;
@@ -454,6 +474,20 @@ export function fromAgUiMessages(
       continue;
     }
 
+    if (role === "reasoning") {
+      // Gate on showThinking so a cold reload matches the live run: the
+      // aggregator never stores reasoning parts when showThinking is false.
+      if (!showThinking) continue;
+      const text = extractText(rawMessage.content);
+      if (text.trim().length === 0) continue;
+      converted.push({
+        id: getString(rawMessage, "id") ?? generateId(),
+        role: "assistant",
+        content: [{ type: "reasoning", text }],
+      });
+      continue;
+    }
+
     if (role === "user" || role === "system") {
       converted.push(toUserOrSystemSnapshotMessage(role, rawMessage));
     }
@@ -478,6 +512,12 @@ function convertAssistantMessage(
     part,
   }));
 
+  // Drop assistant messages with no text or tool calls (e.g. an imported
+  // reasoning-only entry) so they are not re-sent as a blank assistant turn.
+  if (content.length === 0 && toolCalls.length === 0) {
+    return;
+  }
+
   const assistantMessage: AgUiMessage = {
     id: message.id,
     role: "assistant",

package/src/runtime/adapter/run-aggregator.ts

@@ -1,10 +1,11 @@
 "use client";
 
-import type {
-  ChatModelRunResult,
-  MessageTiming,
-  ThreadAssistantMessagePart,
-  ToolCallMessagePart,
+import {
+  isMcpAppUri,
+  type ChatModelRunResult,
+  type MessageTiming,
+  type ThreadAssistantMessagePart,
+  type ToolCallMessagePart,
 } from "@assistant-ui/core";
 import type { AgUiEvent, AgUiInterrupt } from "../types";
 import type { Logger } from "../logger";
@@ -26,8 +27,11 @@ type ToolCallState = {
   isError: boolean | undefined;
   parentMessageId?: string;
   toolMessageId?: string;
+  mcpAppResourceUri?: string;
 };
 
+const MCP_APPS_ACTIVITY_TYPE = "mcp-apps";
+
 export type RunAggregatorOptions = {
   showThinking: boolean;
   logger: Logger;
@@ -58,6 +62,7 @@ export class RunAggregator {
   private activeReasoningKey: string | undefined;
   private reasoningPartCounter = 0;
   private readonly toolCalls = new Map<string, ToolCallState>();
+  private lastResolvedToolCallId: string | undefined;
   private readonly partOrder: (
     | { kind: "text"; key: string }
     | { kind: "reasoning"; key: string }
@@ -85,6 +90,7 @@ export class RunAggregator {
         this.activeReasoningKey = undefined;
         this.reasoningPartCounter = 0;
         this.toolCalls.clear();
+        this.lastResolvedToolCallId = undefined;
         this.partOrder.length = 0;
         this.textPartCounter = 0;
         this.activeTextMessageId = undefined;
@@ -224,6 +230,21 @@ export class RunAggregator {
         this.emit();
         break;
       }
+      case "ACTIVITY_SNAPSHOT": {
+        if (event.activityType !== MCP_APPS_ACTIVITY_TYPE) break;
+        const id = this.lastResolvedToolCallId;
+        const entry = id ? this.toolCalls.get(id) : undefined;
+        const resourceUri = event.content.resourceUri;
+        if (
+          entry &&
+          typeof resourceUri === "string" &&
+          isMcpAppUri(resourceUri)
+        ) {
+          entry.mcpAppResourceUri = resourceUri;
+          this.emit();
+        }
+        break;
+      }
 
       default: {
         this.logger.debug?.("[agui] aggregator ignored event", event);
@@ -372,6 +393,7 @@ export class RunAggregator {
     if (toolMessageId) {
       entry.toolMessageId = toolMessageId;
     }
+    this.lastResolvedToolCallId = id;
   }
 
   private tryParseJSON(value: string): unknown {
@@ -415,6 +437,9 @@ export class RunAggregator {
         argsText: entry.argsText,
         ...(entry.result !== undefined ? { result: entry.result } : {}),
         ...(entry.isError !== undefined ? { isError: entry.isError } : {}),
+        ...(entry.mcpAppResourceUri
+          ? { mcp: { app: { resourceUri: entry.mcpAppResourceUri } } }
+          : {}),
         ...(entry.parentMessageId ? { parentId: entry.parentMessageId } : {}),
         ...(entry.toolMessageId
           ? { unstable_toolMessageId: entry.toolMessageId }

package/src/runtime/adapter/subscriber.ts

@@ -27,6 +27,7 @@ type Subscriber = {
   onToolCallEndEvent?: (payload: { event: unknown }) => void;
   onToolCallChunkEvent?: (payload: { event: unknown }) => void;
   onToolCallResultEvent?: (payload: { event: unknown }) => void;
+  onActivitySnapshotEvent?: (payload: { event: unknown }) => void;
   onStateSnapshotEvent?: (payload: { event: unknown }) => void;
   onStateDeltaEvent?: (payload: { event: unknown }) => void;
   onMessagesSnapshotEvent?: (payload: { event: unknown }) => void;
@@ -122,6 +123,8 @@ export const createAgUiSubscriber = (
       dispatchIfValid(event, "TOOL_CALL_CHUNK"),
     onToolCallResultEvent: ({ event }) =>
       dispatchIfValid(event, "TOOL_CALL_RESULT"),
+    onActivitySnapshotEvent: ({ event }) =>
+      dispatchIfValid(event, "ACTIVITY_SNAPSHOT"),
     onStateSnapshotEvent: ({ event }) =>
       dispatchIfValid(event, "STATE_SNAPSHOT"),
     onStateDeltaEvent: ({ event }) => dispatchIfValid(event, "STATE_DELTA"),

package/src/runtime/event-parser.ts

@@ -235,6 +235,15 @@ export const parseAgUiEvent = (
           ? (payload.messages as any[])
           : [],
       };
+    case "ACTIVITY_SNAPSHOT": {
+      const activityType = getString("activityType");
+      if (!activityType || !isPlainObject(payload.content)) return null;
+      return {
+        type: "ACTIVITY_SNAPSHOT" as const,
+        activityType,
+        content: payload.content,
+      };
+    }
     case "RAW":
       return withOptional(
         { type: "RAW" as const, event: payload.event },

package/src/runtime/types.ts

@@ -121,6 +121,11 @@ export type AgUiEvent =
       content: string;
       role?: "tool";
     }
+  | {
+      type: "ACTIVITY_SNAPSHOT";
+      activityType: string;
+      content: Record<string, unknown>;
+    }
   | { type: "RAW"; event: any; source?: string }
   | { type: "CUSTOM"; name: string; value: any }
   | { type: "STATE_SNAPSHOT"; snapshot: any }