@assistant-ui/core 0.3.18 → 0.3.20

package/src/runtime/utils/external-message-conversion.ts

@@ -1,5 +1,6 @@
 import type { ReadonlyJSONValue } from "assistant-stream/utils";
 import type { ToolExecutionStatus } from "../../runtimes/tool-invocations/ToolInvocationTracker";
+import { ThreadMessageConverter } from "../../runtimes/external-store/thread-message-converter";
 import type {
   ThreadAssistantMessage,
   ThreadMessage,
@@ -11,6 +12,7 @@ import { isJSONValueEqual } from "../../utils/json/is-json-equal";
 import {
   getAutoStatus,
   isAutoStatus,
+  isBackgroundToolCall,
   isInterruptedToolCall,
   isPendingToolCall,
 } from "./auto-status";
@@ -132,6 +134,9 @@ const mergeInnerMessages = (existing: object, incoming: object) => ({
   ],
 });
 
+const isNaNToolCallId = (toolCallId: unknown) =>
+  typeof toolCallId === "number" && Number.isNaN(toolCallId);
+
 export const joinExternalMessages = (
   messages: readonly ExternalMessageConverterMessage[],
 ): ThreadMessageLike => {
@@ -142,11 +147,13 @@ export const joinExternalMessages = (
     role: "assistant",
     content: [],
   };
+  const toolCallIndices = new Map<unknown, number>();
+  const reasoningIndices = new Map<string, number>();
   for (const output of messages) {
     if (output.role === "tool") {
-      const toolCallIdx = assistantMessage.content.findIndex(
-        (c) => c.type === "tool-call" && c.toolCallId === output.toolCallId,
-      );
+      const toolCallIdx = !isNaNToolCallId(output.toolCallId)
+        ? (toolCallIndices.get(output.toolCallId) ?? -1)
+        : -1;
       // Ignore orphaned tool results so one bad tool message does not
       // prevent rendering the rest of the conversation.
       if (toolCallIdx !== -1) {
@@ -239,6 +246,10 @@ export const joinExternalMessages = (
               assistantMessage.metadata.timing = output.metadata.timing;
             }
 
+            if (output.metadata.modality) {
+              assistantMessage.metadata.modality = output.metadata.modality;
+            }
+
             if (output.metadata.submittedFeedback) {
               assistantMessage.metadata.submittedFeedback =
                 output.metadata.submittedFeedback;
@@ -253,11 +264,8 @@ export const joinExternalMessages = (
           // Add content parts, merging reasoning parts with same parentId
           for (const part of content) {
             if (part.type === "tool-call" && part.toolCallId) {
-              const existingIdx = assistantMessage.content.findIndex(
-                (c) =>
-                  c.type === "tool-call" && c.toolCallId === part.toolCallId,
-              );
-              if (existingIdx !== -1) {
+              const existingIdx = toolCallIndices.get(part.toolCallId);
+              if (existingIdx !== undefined) {
                 const existing = assistantMessage.content[
                   existingIdx
                 ] as typeof part;
@@ -275,13 +283,8 @@ export const joinExternalMessages = (
               "parentId" in part &&
               part.parentId
             ) {
-              const existingIdx = assistantMessage.content.findIndex(
-                (c) =>
-                  c.type === "reasoning" &&
-                  "parentId" in c &&
-                  c.parentId === part.parentId,
-              );
-              if (existingIdx !== -1) {
+              const existingIdx = reasoningIndices.get(part.parentId);
+              if (existingIdx !== undefined) {
                 const existing = assistantMessage.content[
                   existingIdx
                 ] as typeof part;
@@ -293,7 +296,21 @@ export const joinExternalMessages = (
                 continue;
               }
             }
+            const partIdx = assistantMessage.content.length;
             assistantMessage.content.push(part);
+            if (
+              part.type === "tool-call" &&
+              !isNaNToolCallId(part.toolCallId) &&
+              !toolCallIndices.has(part.toolCallId)
+            ) {
+              toolCallIndices.set(part.toolCallId, partIdx);
+            } else if (
+              part.type === "reasoning" &&
+              "parentId" in part &&
+              part.parentId
+            ) {
+              reasoningIndices.set(part.parentId, partIdx);
+            }
           }
           break;
         default: {
@@ -331,11 +348,14 @@ export const chunkExternalMessages = <T>(
 
   for (const callbackResult of callbackResults) {
     for (const output of callbackResult.outputs) {
+      const isVoice =
+        output.role === "assistant" && output.metadata?.modality === "voice";
       if (
         (pendingNone && output.role !== "tool") ||
         !isAssistant ||
         output.role === "user" ||
-        output.role === "system"
+        output.role === "system" ||
+        isVoice
       ) {
         flush();
       }
@@ -348,7 +368,8 @@ export const chunkExternalMessages = <T>(
 
       if (
         output.role === "assistant" &&
-        (output.convertConfig?.joinStrategy === "none" ||
+        (isVoice ||
+          output.convertConfig?.joinStrategy === "none" ||
           joinStrategy === "none")
       ) {
         pendingNone = true;
@@ -370,18 +391,46 @@ export const shallowArrayEqual = (
   return true;
 };
 
-type ExternalMessageConversionCache = {
+type ExternalMessageChunkConversionCache = {
   message: ThreadMessage | undefined;
   generatedFallbackMessages: WeakSet<object>;
 };
 
+type ExternalMessageConversionCallbackCacheEntry<T> =
+  ExternalMessageConverterCallbackResult<T> & {
+    metadata: ExternalMessageConverterMetadata;
+    callback: ExternalMessageConverterCallback<T>;
+  };
+
+export type InternalExternalMessageConversionCache<
+  T extends WeakKey = WeakKey,
+> = {
+  callbackCache: WeakMap<T, ExternalMessageConversionCallbackCacheEntry<T>>;
+  chunkCache: WeakMap<
+    ExternalMessageConverterMessage,
+    ExternalMessageConverterChunk<T>
+  >;
+  converterCache: ThreadMessageConverter;
+  // Generatedness is tracked by identity, not by id shape: a caller-supplied id that happens to match the generated pattern must never be rewritten.
+  generatedFallbackMessages: WeakSet<object>;
+};
+
+export const createExternalMessageConversionCache = <
+  T extends WeakKey = WeakKey,
+>(): InternalExternalMessageConversionCache<T> => ({
+  callbackCache: new WeakMap(),
+  chunkCache: new WeakMap(),
+  converterCache: new ThreadMessageConverter(),
+  generatedFallbackMessages: new WeakSet(),
+});
+
 export const convertExternalMessageChunk = <T>(
   message: ExternalMessageConverterChunk<T>,
   idx: number,
   chunkCount: number,
   isRunning: boolean,
   error: ReadonlyJSONValue | undefined,
-  cache?: ExternalMessageConversionCache,
+  cache?: ExternalMessageChunkConversionCache,
   cancelledMessageIds?: ReadonlySet<string>,
 ) => {
   const isLast = idx === chunkCount - 1;
@@ -400,6 +449,9 @@ export const convertExternalMessageChunk = <T>(
   const hasPendingToolCalls =
     typeof joined.content === "object" &&
     joined.content.some(isPendingToolCall);
+  const hasBackgroundToolCalls =
+    typeof joined.content === "object" &&
+    joined.content.some(isBackgroundToolCall);
   const autoStatus = getAutoStatus(
     isLast,
     isRunning,
@@ -407,6 +459,7 @@ export const convertExternalMessageChunk = <T>(
     hasPendingToolCalls,
     isLast ? error : undefined,
     isCancelled,
+    hasBackgroundToolCalls,
   );
   const fallbackId = `${FALLBACK_ID_PREFIX}${idx}`;
 
@@ -490,21 +543,63 @@ export const convertExternalMessages = <T extends WeakKey>(
   callback: ExternalMessageConverterCallback<T>,
   isRunning: boolean,
   metadata: ExternalMessageConverterMetadata,
+  joinStrategy?: JoinStrategy,
+  cache?: InternalExternalMessageConversionCache<T>,
 ) => {
-  const callbackResults = messages.map((message) =>
-    convertExternalMessageCallback(message, callback, metadata),
-  );
-  const chunks = chunkExternalMessages(callbackResults);
-  const result = chunks.map((message, idx) =>
-    convertExternalMessageChunk(
-      message,
-      idx,
-      chunks.length,
-      isRunning,
-      metadata.error,
-      undefined,
-      metadata.cancelledMessageIds,
-    ),
+  const callbackResults = messages.map((message) => {
+    let result = cache?.callbackCache.get(message);
+    if (
+      !result ||
+      result.metadata !== metadata ||
+      result.callback !== callback
+    ) {
+      result = {
+        ...convertExternalMessageCallback(message, callback, metadata),
+        metadata,
+        callback,
+      };
+      cache?.callbackCache.set(message, result);
+    }
+    return result;
+  });
+  const chunks = chunkExternalMessages(callbackResults, joinStrategy).map(
+    (message) => {
+      const key = message.outputs[0];
+      if (!key || !cache) return message;
+
+      const cached = cache.chunkCache.get(key);
+      if (cached && shallowArrayEqual(cached.outputs, message.outputs)) {
+        return cached;
+      }
+      cache.chunkCache.set(key, message);
+      return message;
+    },
   );
+  const result = cache
+    ? cache.converterCache.convertMessages(chunks, (cached, message, idx) =>
+        convertExternalMessageChunk(
+          message,
+          idx,
+          chunks.length,
+          isRunning,
+          metadata.error,
+          {
+            message: cached,
+            generatedFallbackMessages: cache.generatedFallbackMessages,
+          },
+          metadata.cancelledMessageIds,
+        ),
+      )
+    : chunks.map((message, idx) =>
+        convertExternalMessageChunk(
+          message,
+          idx,
+          chunks.length,
+          isRunning,
+          metadata.error,
+          undefined,
+          metadata.cancelledMessageIds,
+        ),
+      );
   return completeExternalMessageConversion(result, metadata.error);
 };

package/src/runtime/utils/message-repository.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+import type { ThreadMessage } from "../../types/message";
+import { ExportedMessageRepository } from "./message-repository";
+
+const nestedRunningAssistant: ThreadMessage = {
+  id: "nested-assistant",
+  createdAt: new Date(0),
+  role: "assistant",
+  content: [],
+  status: { type: "running" },
+  metadata: {
+    unstable_state: {},
+    unstable_annotations: [],
+    unstable_data: [],
+    steps: [],
+    custom: {},
+  },
+};
+
+const delegating = {
+  id: "assistant-1",
+  role: "assistant" as const,
+  content: [
+    {
+      type: "tool-call" as const,
+      toolCallId: "delegate-1",
+      toolName: "delegate",
+      args: {},
+      argsText: "",
+      messages: [nestedRunningAssistant],
+    },
+  ],
+};
+
+describe("ExportedMessageRepository", () => {
+  it("imports a message saved mid-delegation as pending, not running", () => {
+    const fromArray = ExportedMessageRepository.fromArray([delegating]);
+    const fromBranchable = ExportedMessageRepository.fromBranchableArray([
+      { message: delegating, parentId: null },
+    ]);
+
+    for (const repository of [fromArray, fromBranchable]) {
+      expect(repository.messages[0]?.message.status).toMatchObject({
+        type: "requires-action",
+        reason: "tool-calls",
+      });
+    }
+  });
+});

package/src/runtime/utils/message-repository.ts

@@ -2,7 +2,7 @@ import type { ThreadMessage } from "../../types/message";
 import type { RunConfig } from "../../types/message";
 import { generateId } from "../../utils/id";
 import type { ThreadMessageLike } from "./thread-message-like";
-import { getContentAutoStatus } from "./auto-status";
+import { getRepositoryContentAutoStatus } from "./auto-status";
 import { fromThreadMessageLike } from "./thread-message-like";
 
 export type ExportedMessageRepositoryItem = {
@@ -28,7 +28,7 @@ export const ExportedMessageRepository = {
       fromThreadMessageLike(
         m,
         generateId(),
-        getContentAutoStatus(m.content, false, false),
+        getRepositoryContentAutoStatus(m.content),
       ),
     );
 
@@ -62,7 +62,7 @@ export const ExportedMessageRepository = {
           message: fromThreadMessageLike(
             message,
             message.id,
-            getContentAutoStatus(message.content, false, false),
+            getRepositoryContentAutoStatus(message.content),
           ),
         };
       }),

package/src/runtime/utils/stored-message-parts.ts

@@ -0,0 +1,174 @@
+import type { CompleteAttachment } from "../../types/attachment";
+import type {
+  ThreadAssistantMessagePart,
+  ThreadMessage,
+  ThreadUserMessagePart,
+} from "../../types/message";
+import { isRecord } from "../../utils/json/is-json";
+
+export const MAX_STORED_MESSAGE_DEPTH = 100;
+
+type StoredPartGuard = (part: Record<string, unknown>) => boolean;
+
+export type StoredMessagePart = Record<string, unknown> & { type: string };
+
+export const parseStoredDate = (value: unknown): Date | null => {
+  if (value instanceof Date && !Number.isNaN(value.getTime())) return value;
+  if (typeof value !== "string" && typeof value !== "number") return null;
+
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? null : date;
+};
+
+export const isStoredMessageRole = (
+  value: unknown,
+): value is ThreadMessage["role"] =>
+  value === "system" || value === "user" || value === "assistant";
+
+/**
+ * Builds the readability predicate for a persistence boundary. A stored part is
+ * readable when it passes the guard for its known type, or when its type is
+ * unknown and the boundary can still read it.
+ *
+ * Unknown types are where the boundaries part: local storage hands its rows to
+ * the runtime untouched, so a type written by a newer release still loads,
+ * while the aui/v0 decoder converts a `data-` prefix and throws on anything
+ * else unknown. Per-type differences ride in `overrides`.
+ */
+const makeIsStoredMessagePart = (
+  overrides: Partial<
+    Record<
+      (ThreadUserMessagePart | ThreadAssistantMessagePart)["type"],
+      StoredPartGuard
+    >
+  >,
+  isReadableUnknownType: (type: string) => boolean,
+) => {
+  const storedPartGuards = {
+    text: (part) => typeof part.text === "string",
+    reasoning: (part) =>
+      typeof part.text === "string" ||
+      typeof part.unstable_summary === "string",
+    image: (part) => typeof part.image === "string",
+    file: (part) =>
+      typeof part.data === "string" && typeof part.mimeType === "string",
+    audio: (part) =>
+      isRecord(part.audio) &&
+      typeof part.audio.data === "string" &&
+      typeof part.audio.format === "string",
+    data: (part) => typeof part.name === "string",
+    source: (part) =>
+      typeof part.id === "string" &&
+      (part.sourceType === "url"
+        ? typeof part.url === "string"
+        : part.sourceType === "document" &&
+          typeof part.title === "string" &&
+          typeof part.mediaType === "string"),
+    "generative-ui": (part) => isRecord(part.spec),
+    "tool-call": (part) =>
+      typeof part.toolCallId === "string" &&
+      typeof part.toolName === "string" &&
+      isRecord(part.args) &&
+      typeof part.argsText === "string",
+  } satisfies Record<
+    (ThreadUserMessagePart | ThreadAssistantMessagePart)["type"],
+    StoredPartGuard
+  >;
+  const guards: Record<string, StoredPartGuard> = {
+    ...storedPartGuards,
+    ...overrides,
+  };
+
+  return (value: unknown): value is StoredMessagePart =>
+    isRecord(value) &&
+    typeof value.type === "string" &&
+    (Object.hasOwn(guards, value.type)
+      ? guards[value.type]!(value)
+      : isReadableUnknownType(value.type));
+};
+
+export const isStoredMessagePart = makeIsStoredMessagePart({}, () => true);
+
+/**
+ * The aui/v0 decoder dereferences each present field rather than reading only
+ * the one it needs, so this boundary checks every field it will touch, while
+ * local storage stays no stricter than what the runtime itself writes.
+ */
+const isStoredAuiV0Part = makeIsStoredMessagePart(
+  {
+    reasoning: (part) =>
+      (part.text === undefined || typeof part.text === "string") &&
+      (part.unstable_summary === undefined ||
+        typeof part.unstable_summary === "string") &&
+      (typeof part.text === "string" ||
+        typeof part.unstable_summary === "string"),
+    "tool-call": (part) =>
+      typeof part.toolCallId === "string" &&
+      typeof part.toolName === "string" &&
+      (part.args === undefined || isRecord(part.args)) &&
+      (part.argsText === undefined || typeof part.argsText === "string") &&
+      (part.args !== undefined || part.argsText !== undefined),
+  },
+  (type) => type.startsWith("data-"),
+);
+
+const rolePartTypes = {
+  user: {
+    text: true,
+    image: true,
+    file: true,
+    data: true,
+    audio: true,
+  } satisfies Record<ThreadUserMessagePart["type"], true>,
+  assistant: {
+    text: true,
+    reasoning: true,
+    "tool-call": true,
+    source: true,
+    file: true,
+    image: true,
+    data: true,
+    "generative-ui": true,
+  } satisfies Record<ThreadAssistantMessagePart["type"], true>,
+};
+
+/**
+ * `fromThreadMessageLike` converts each part through its role's branch and
+ * throws on a part that branch does not carry, which would cost the whole
+ * stored row. The aui/v0 boundary decodes through that converter, so it filters
+ * by role as well as by shape. A part type unknown to this table is left to the
+ * boundary predicate, which keeps the `data-` prefixes the converter handles.
+ */
+export const isStoredAuiV0RolePart = (
+  role: ThreadMessage["role"],
+  value: unknown,
+): value is StoredMessagePart =>
+  isStoredAuiV0Part(value) &&
+  (role === "system"
+    ? value.type === "text"
+    : Object.hasOwn(rolePartTypes.assistant, value.type) ||
+        Object.hasOwn(rolePartTypes.user, value.type)
+      ? Object.hasOwn(rolePartTypes[role], value.type)
+      : true);
+
+export const parseStoredAttachment = (
+  value: unknown,
+  isPart: (value: unknown) => value is StoredMessagePart,
+): CompleteAttachment | null => {
+  if (
+    !isRecord(value) ||
+    typeof value.id !== "string" ||
+    typeof value.type !== "string" ||
+    typeof value.name !== "string" ||
+    !isRecord(value.status) ||
+    value.status.type !== "complete" ||
+    !Array.isArray(value.content)
+  ) {
+    return null;
+  }
+
+  return {
+    ...value,
+    content: value.content.filter(isPart),
+  } as CompleteAttachment;
+};

package/src/runtime/utils/thread-message-like.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import { fromThreadMessageLike } from "./thread-message-like";
+
+describe("fromThreadMessageLike", () => {
+  it("preserves modality on user and assistant messages", () => {
+    const user = fromThreadMessageLike(
+      {
+        role: "user",
+        content: "Hello",
+        metadata: { modality: "voice" },
+      },
+      "user-id",
+      { type: "complete", reason: "unknown" },
+    );
+    const assistant = fromThreadMessageLike(
+      {
+        role: "assistant",
+        content: "Hi",
+        metadata: { modality: "voice" },
+      },
+      "assistant-id",
+      { type: "complete", reason: "unknown" },
+    );
+
+    expect(user.metadata.modality).toBe("voice");
+    expect(assistant.metadata.modality).toBe("voice");
+  });
+
+  it("leaves modality absent when the input has none", () => {
+    const message = fromThreadMessageLike(
+      { role: "user", content: "Hello" },
+      "user-id",
+      { type: "complete", reason: "unknown" },
+    );
+
+    expect(message.metadata).not.toHaveProperty("modality");
+  });
+
+  it("ignores modality on system messages", () => {
+    const message = fromThreadMessageLike(
+      {
+        role: "system",
+        content: "Instructions",
+        metadata: { modality: "voice" },
+      },
+      "system-id",
+      { type: "complete", reason: "unknown" },
+    );
+
+    expect(message.metadata).not.toHaveProperty("modality");
+  });
+});

package/src/runtime/utils/thread-message-like.ts

@@ -20,6 +20,7 @@ import type {
 } from "../../types/message";
 import type { CompleteAttachment } from "../../types/attachment";
 import type {
+  MessageModality,
   MessageTiming,
   PartProviderMetadata,
   TextMessagePart,
@@ -79,15 +80,21 @@ export type ThreadMessageLike = {
     | undefined;
   readonly metadata?:
     | {
-        readonly unstable_state?: ReadonlyJSONValue;
+        readonly unstable_state?: ReadonlyJSONValue | undefined;
         readonly unstable_annotations?:
           | readonly ReadonlyJSONValue[]
           | undefined;
         readonly unstable_data?: readonly ReadonlyJSONValue[] | undefined;
         readonly steps?: readonly ThreadStep[] | undefined;
         readonly timing?: MessageTiming | undefined;
-        readonly submittedFeedback?: { readonly type: "positive" | "negative" };
+        readonly submittedFeedback?:
+          | {
+              readonly type: "positive" | "negative";
+              readonly comment?: string;
+            }
+          | undefined;
         readonly isOptimistic?: boolean | undefined;
+        readonly modality?: MessageModality | undefined;
         readonly custom?: Record<string, unknown> | undefined;
       }
     | undefined;
@@ -223,6 +230,7 @@ export const fromThreadMessageLike = (
             submittedFeedback: metadata.submittedFeedback,
           }),
           ...(metadata?.isOptimistic && { isOptimistic: true }),
+          ...(metadata?.modality && { modality: metadata.modality }),
         },
       } satisfies ThreadAssistantMessage;
 
@@ -263,6 +271,7 @@ export const fromThreadMessageLike = (
         metadata: {
           custom: metadata?.custom ?? {},
           ...(metadata?.isOptimistic && { isOptimistic: true }),
+          ...(metadata?.modality && { modality: metadata.modality }),
         },
       } satisfies ThreadUserMessage;
 

package/src/runtimes/external-store/external-store-adapter.ts

@@ -133,6 +133,10 @@ type ExternalStoreAdapterBase<T> = {
    * back, because the runtime cannot see a removal it did not make.
    */
   setMessages?: ((messages: readonly T[]) => void) | undefined;
+  /**
+   * Called with each finalized voice transcript message. The host appends it to its own messages under the same id, which is how the runtime knows the host carries it. A host that does not implement this callback keeps transcripts for the session only.
+   */
+  onVoiceTranscript?: ((message: ThreadMessage) => void) | undefined;
   /**
    * Fires when the user explicitly switches branches via the runtime's
    * `switchToBranch` action (e.g. a BranchPicker click). It does not fire on