@assistant-ui/core 0.3.18 → 0.3.19

package/src/runtime/utils/auto-status.ts

@@ -10,6 +10,15 @@ type ThreadMessageLikeContentItem = Exclude<
 export const isPendingToolCall = (c: ThreadMessageLikeContentItem): boolean =>
   c.type === "tool-call" && c.result === undefined;
 
+/** A pending tool call whose nested conversation is still streaming; the outer message stays running until the nested run settles. */
+export const isBackgroundToolCall = (
+  c: ThreadMessageLikeContentItem,
+): boolean => {
+  if (c.type !== "tool-call" || c.result !== undefined) return false;
+  const last = c.messages?.at(-1);
+  return last?.role === "assistant" && last.status.type === "running";
+};
+
 export const isInterruptedToolCall = (
   c: ThreadMessageLikeContentItem,
 ): boolean => {
@@ -76,6 +85,7 @@ export const getAutoStatus = (
   hasPendingToolCalls: boolean,
   error?: ReadonlyJSONValue,
   isCancelled?: boolean,
+  hasBackgroundToolCalls?: boolean,
 ): MessageStatus => {
   if (isLast && error) {
     return Object.assign(
@@ -92,13 +102,26 @@ export const getAutoStatus = (
     ? AUTO_STATUS_RUNNING
     : hasInterruptedToolCalls
       ? AUTO_STATUS_INTERRUPT
-      : hasPendingToolCalls
-        ? AUTO_STATUS_PENDING
-        : isCancelled
-          ? AUTO_STATUS_CANCELLED
-          : AUTO_STATUS_COMPLETE;
+      : hasBackgroundToolCalls && !isCancelled
+        ? AUTO_STATUS_RUNNING
+        : hasPendingToolCalls
+          ? AUTO_STATUS_PENDING
+          : isCancelled
+            ? AUTO_STATUS_CANCELLED
+            : AUTO_STATUS_COMPLETE;
 };
 
+/** Status for a message entering the repository: the nested conversation is not being fed here, so it never keeps the message running. */
+export const getRepositoryContentAutoStatus = (
+  content: ThreadMessageLike["content"],
+): MessageStatus =>
+  getAutoStatus(
+    false,
+    false,
+    typeof content !== "string" && content.some(isInterruptedToolCall),
+    typeof content !== "string" && content.some(isPendingToolCall),
+  );
+
 export const getContentAutoStatus = (
   content: ThreadMessageLike["content"],
   isLast: boolean,
@@ -109,4 +132,7 @@ export const getContentAutoStatus = (
     isRunning,
     typeof content !== "string" && content.some(isInterruptedToolCall),
     typeof content !== "string" && content.some(isPendingToolCall),
+    undefined,
+    undefined,
+    typeof content !== "string" && content.some(isBackgroundToolCall),
   );

package/src/runtime/utils/external-message-conversion.test.ts

@@ -1,10 +1,13 @@
 import { describe, expect, it, vi } from "vitest";
+import type { ThreadMessage } from "../../types/message";
 import {
   chunkExternalMessages,
   convertExternalMessageCallback,
   convertExternalMessageChunk,
+  joinExternalMessages,
   type ExternalMessageConverterCallback,
   type ExternalMessageConverterCallbackResult,
+  type ExternalMessageConverterMessage,
 } from "./external-message-conversion";
 
 describe("convertExternalMessageCallback", () => {
@@ -30,6 +33,75 @@ describe("convertExternalMessageCallback", () => {
   });
 });
 
+describe("joinExternalMessages", () => {
+  it("preserves strict equality for malformed numeric tool-call IDs", () => {
+    const messages = [
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "tool-call",
+            toolCallId: 1,
+            toolName: "search",
+            args: { query: "old" },
+          },
+        ],
+      },
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "tool-call",
+            toolCallId: 1,
+            toolName: "search",
+            args: { query: "new" },
+          },
+        ],
+      },
+    ] as unknown as ExternalMessageConverterMessage[];
+
+    expect(joinExternalMessages(messages).content).toMatchObject([
+      {
+        type: "tool-call",
+        toolCallId: 1,
+        args: { query: "new" },
+      },
+    ]);
+  });
+
+  it("does not merge malformed NaN tool-call IDs", () => {
+    const messages = [
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "tool-call",
+            toolCallId: Number.NaN,
+            toolName: "search",
+            args: { query: "first" },
+          },
+          {
+            type: "tool-call",
+            toolCallId: Number.NaN,
+            toolName: "search",
+            args: { query: "second" },
+          },
+        ],
+      },
+      { role: "tool", toolCallId: Number.NaN, result: "found" },
+    ] as unknown as ExternalMessageConverterMessage[];
+
+    const result = joinExternalMessages(messages);
+
+    expect(result.content).toMatchObject([
+      { type: "tool-call", args: { query: "first" } },
+      { type: "tool-call", args: { query: "second" } },
+    ]);
+    expect(result.content[0]).not.toHaveProperty("result");
+    expect(result.content[1]).not.toHaveProperty("result");
+  });
+});
+
 describe("chunkExternalMessages", () => {
   it("keeps a tool result with a non-joining assistant before starting the next chunk", () => {
     const toolCall = {};
@@ -83,6 +155,68 @@ describe("chunkExternalMessages", () => {
 });
 
 describe("convertExternalMessageChunk", () => {
+  it("keeps an ended outer run running for a background tool call", () => {
+    const nestedAssistant: ThreadMessage = {
+      id: "nested-assistant",
+      createdAt: new Date(0),
+      role: "assistant",
+      content: [],
+      status: { type: "running" },
+      metadata: {
+        unstable_state: {},
+        unstable_annotations: [],
+        unstable_data: [],
+        steps: [],
+        custom: {},
+      },
+    };
+    const result = convertExternalMessageChunk(
+      {
+        inputs: [{}],
+        outputs: [
+          {
+            role: "assistant",
+            content: [
+              {
+                type: "tool-call",
+                toolName: "delegate",
+                args: {},
+                messages: [nestedAssistant],
+              },
+            ],
+          },
+        ],
+      },
+      0,
+      1,
+      false,
+      undefined,
+    );
+
+    expect(result.status).toMatchObject({ type: "running" });
+  });
+
+  it("preserves assistant message modality", () => {
+    const result = convertExternalMessageChunk(
+      {
+        inputs: [{}],
+        outputs: [
+          {
+            role: "assistant",
+            content: "Spoken reply",
+            metadata: { modality: "voice" },
+          },
+        ],
+      },
+      0,
+      1,
+      false,
+      undefined,
+    );
+
+    expect(result.metadata.modality).toBe("voice");
+  });
+
   it("keeps separate tool calls without IDs", () => {
     const result = convertExternalMessageChunk(
       {

package/src/runtime/utils/external-message-conversion.ts

@@ -1,5 +1,6 @@
 import type { ReadonlyJSONValue } from "assistant-stream/utils";
 import type { ToolExecutionStatus } from "../../runtimes/tool-invocations/ToolInvocationTracker";
+import { ThreadMessageConverter } from "../../runtimes/external-store/thread-message-converter";
 import type {
   ThreadAssistantMessage,
   ThreadMessage,
@@ -11,6 +12,7 @@ import { isJSONValueEqual } from "../../utils/json/is-json-equal";
 import {
   getAutoStatus,
   isAutoStatus,
+  isBackgroundToolCall,
   isInterruptedToolCall,
   isPendingToolCall,
 } from "./auto-status";
@@ -132,6 +134,9 @@ const mergeInnerMessages = (existing: object, incoming: object) => ({
   ],
 });
 
+const isNaNToolCallId = (toolCallId: unknown) =>
+  typeof toolCallId === "number" && Number.isNaN(toolCallId);
+
 export const joinExternalMessages = (
   messages: readonly ExternalMessageConverterMessage[],
 ): ThreadMessageLike => {
@@ -142,11 +147,13 @@ export const joinExternalMessages = (
     role: "assistant",
     content: [],
   };
+  const toolCallIndices = new Map<unknown, number>();
+  const reasoningIndices = new Map<string, number>();
   for (const output of messages) {
     if (output.role === "tool") {
-      const toolCallIdx = assistantMessage.content.findIndex(
-        (c) => c.type === "tool-call" && c.toolCallId === output.toolCallId,
-      );
+      const toolCallIdx = !isNaNToolCallId(output.toolCallId)
+        ? (toolCallIndices.get(output.toolCallId) ?? -1)
+        : -1;
       // Ignore orphaned tool results so one bad tool message does not
       // prevent rendering the rest of the conversation.
       if (toolCallIdx !== -1) {
@@ -239,6 +246,10 @@ export const joinExternalMessages = (
               assistantMessage.metadata.timing = output.metadata.timing;
             }
 
+            if (output.metadata.modality) {
+              assistantMessage.metadata.modality = output.metadata.modality;
+            }
+
             if (output.metadata.submittedFeedback) {
               assistantMessage.metadata.submittedFeedback =
                 output.metadata.submittedFeedback;
@@ -253,11 +264,8 @@ export const joinExternalMessages = (
           // Add content parts, merging reasoning parts with same parentId
           for (const part of content) {
             if (part.type === "tool-call" && part.toolCallId) {
-              const existingIdx = assistantMessage.content.findIndex(
-                (c) =>
-                  c.type === "tool-call" && c.toolCallId === part.toolCallId,
-              );
-              if (existingIdx !== -1) {
+              const existingIdx = toolCallIndices.get(part.toolCallId);
+              if (existingIdx !== undefined) {
                 const existing = assistantMessage.content[
                   existingIdx
                 ] as typeof part;
@@ -275,13 +283,8 @@ export const joinExternalMessages = (
               "parentId" in part &&
               part.parentId
             ) {
-              const existingIdx = assistantMessage.content.findIndex(
-                (c) =>
-                  c.type === "reasoning" &&
-                  "parentId" in c &&
-                  c.parentId === part.parentId,
-              );
-              if (existingIdx !== -1) {
+              const existingIdx = reasoningIndices.get(part.parentId);
+              if (existingIdx !== undefined) {
                 const existing = assistantMessage.content[
                   existingIdx
                 ] as typeof part;
@@ -293,7 +296,21 @@ export const joinExternalMessages = (
                 continue;
               }
             }
+            const partIdx = assistantMessage.content.length;
             assistantMessage.content.push(part);
+            if (
+              part.type === "tool-call" &&
+              !isNaNToolCallId(part.toolCallId) &&
+              !toolCallIndices.has(part.toolCallId)
+            ) {
+              toolCallIndices.set(part.toolCallId, partIdx);
+            } else if (
+              part.type === "reasoning" &&
+              "parentId" in part &&
+              part.parentId
+            ) {
+              reasoningIndices.set(part.parentId, partIdx);
+            }
           }
           break;
         default: {
@@ -370,18 +387,46 @@ export const shallowArrayEqual = (
   return true;
 };
 
-type ExternalMessageConversionCache = {
+type ExternalMessageChunkConversionCache = {
   message: ThreadMessage | undefined;
   generatedFallbackMessages: WeakSet<object>;
 };
 
+type ExternalMessageConversionCallbackCacheEntry<T> =
+  ExternalMessageConverterCallbackResult<T> & {
+    metadata: ExternalMessageConverterMetadata;
+    callback: ExternalMessageConverterCallback<T>;
+  };
+
+export type InternalExternalMessageConversionCache<
+  T extends WeakKey = WeakKey,
+> = {
+  callbackCache: WeakMap<T, ExternalMessageConversionCallbackCacheEntry<T>>;
+  chunkCache: WeakMap<
+    ExternalMessageConverterMessage,
+    ExternalMessageConverterChunk<T>
+  >;
+  converterCache: ThreadMessageConverter;
+  // Generatedness is tracked by identity, not by id shape: a caller-supplied id that happens to match the generated pattern must never be rewritten.
+  generatedFallbackMessages: WeakSet<object>;
+};
+
+export const createExternalMessageConversionCache = <
+  T extends WeakKey = WeakKey,
+>(): InternalExternalMessageConversionCache<T> => ({
+  callbackCache: new WeakMap(),
+  chunkCache: new WeakMap(),
+  converterCache: new ThreadMessageConverter(),
+  generatedFallbackMessages: new WeakSet(),
+});
+
 export const convertExternalMessageChunk = <T>(
   message: ExternalMessageConverterChunk<T>,
   idx: number,
   chunkCount: number,
   isRunning: boolean,
   error: ReadonlyJSONValue | undefined,
-  cache?: ExternalMessageConversionCache,
+  cache?: ExternalMessageChunkConversionCache,
   cancelledMessageIds?: ReadonlySet<string>,
 ) => {
   const isLast = idx === chunkCount - 1;
@@ -400,6 +445,9 @@ export const convertExternalMessageChunk = <T>(
   const hasPendingToolCalls =
     typeof joined.content === "object" &&
     joined.content.some(isPendingToolCall);
+  const hasBackgroundToolCalls =
+    typeof joined.content === "object" &&
+    joined.content.some(isBackgroundToolCall);
   const autoStatus = getAutoStatus(
     isLast,
     isRunning,
@@ -407,6 +455,7 @@ export const convertExternalMessageChunk = <T>(
     hasPendingToolCalls,
     isLast ? error : undefined,
     isCancelled,
+    hasBackgroundToolCalls,
   );
   const fallbackId = `${FALLBACK_ID_PREFIX}${idx}`;
 
@@ -490,21 +539,63 @@ export const convertExternalMessages = <T extends WeakKey>(
   callback: ExternalMessageConverterCallback<T>,
   isRunning: boolean,
   metadata: ExternalMessageConverterMetadata,
+  joinStrategy?: JoinStrategy,
+  cache?: InternalExternalMessageConversionCache<T>,
 ) => {
-  const callbackResults = messages.map((message) =>
-    convertExternalMessageCallback(message, callback, metadata),
-  );
-  const chunks = chunkExternalMessages(callbackResults);
-  const result = chunks.map((message, idx) =>
-    convertExternalMessageChunk(
-      message,
-      idx,
-      chunks.length,
-      isRunning,
-      metadata.error,
-      undefined,
-      metadata.cancelledMessageIds,
-    ),
+  const callbackResults = messages.map((message) => {
+    let result = cache?.callbackCache.get(message);
+    if (
+      !result ||
+      result.metadata !== metadata ||
+      result.callback !== callback
+    ) {
+      result = {
+        ...convertExternalMessageCallback(message, callback, metadata),
+        metadata,
+        callback,
+      };
+      cache?.callbackCache.set(message, result);
+    }
+    return result;
+  });
+  const chunks = chunkExternalMessages(callbackResults, joinStrategy).map(
+    (message) => {
+      const key = message.outputs[0];
+      if (!key || !cache) return message;
+
+      const cached = cache.chunkCache.get(key);
+      if (cached && shallowArrayEqual(cached.outputs, message.outputs)) {
+        return cached;
+      }
+      cache.chunkCache.set(key, message);
+      return message;
+    },
   );
+  const result = cache
+    ? cache.converterCache.convertMessages(chunks, (cached, message, idx) =>
+        convertExternalMessageChunk(
+          message,
+          idx,
+          chunks.length,
+          isRunning,
+          metadata.error,
+          {
+            message: cached,
+            generatedFallbackMessages: cache.generatedFallbackMessages,
+          },
+          metadata.cancelledMessageIds,
+        ),
+      )
+    : chunks.map((message, idx) =>
+        convertExternalMessageChunk(
+          message,
+          idx,
+          chunks.length,
+          isRunning,
+          metadata.error,
+          undefined,
+          metadata.cancelledMessageIds,
+        ),
+      );
   return completeExternalMessageConversion(result, metadata.error);
 };

package/src/runtime/utils/message-repository.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+import type { ThreadMessage } from "../../types/message";
+import { ExportedMessageRepository } from "./message-repository";
+
+const nestedRunningAssistant: ThreadMessage = {
+  id: "nested-assistant",
+  createdAt: new Date(0),
+  role: "assistant",
+  content: [],
+  status: { type: "running" },
+  metadata: {
+    unstable_state: {},
+    unstable_annotations: [],
+    unstable_data: [],
+    steps: [],
+    custom: {},
+  },
+};
+
+const delegating = {
+  id: "assistant-1",
+  role: "assistant" as const,
+  content: [
+    {
+      type: "tool-call" as const,
+      toolCallId: "delegate-1",
+      toolName: "delegate",
+      args: {},
+      argsText: "",
+      messages: [nestedRunningAssistant],
+    },
+  ],
+};
+
+describe("ExportedMessageRepository", () => {
+  it("imports a message saved mid-delegation as pending, not running", () => {
+    const fromArray = ExportedMessageRepository.fromArray([delegating]);
+    const fromBranchable = ExportedMessageRepository.fromBranchableArray([
+      { message: delegating, parentId: null },
+    ]);
+
+    for (const repository of [fromArray, fromBranchable]) {
+      expect(repository.messages[0]?.message.status).toMatchObject({
+        type: "requires-action",
+        reason: "tool-calls",
+      });
+    }
+  });
+});

package/src/runtime/utils/message-repository.ts

@@ -2,7 +2,7 @@ import type { ThreadMessage } from "../../types/message";
 import type { RunConfig } from "../../types/message";
 import { generateId } from "../../utils/id";
 import type { ThreadMessageLike } from "./thread-message-like";
-import { getContentAutoStatus } from "./auto-status";
+import { getRepositoryContentAutoStatus } from "./auto-status";
 import { fromThreadMessageLike } from "./thread-message-like";
 
 export type ExportedMessageRepositoryItem = {
@@ -28,7 +28,7 @@ export const ExportedMessageRepository = {
       fromThreadMessageLike(
         m,
         generateId(),
-        getContentAutoStatus(m.content, false, false),
+        getRepositoryContentAutoStatus(m.content),
       ),
     );
 
@@ -62,7 +62,7 @@ export const ExportedMessageRepository = {
           message: fromThreadMessageLike(
             message,
             message.id,
-            getContentAutoStatus(message.content, false, false),
+            getRepositoryContentAutoStatus(message.content),
           ),
         };
       }),

package/src/runtime/utils/thread-message-like.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import { fromThreadMessageLike } from "./thread-message-like";
+
+describe("fromThreadMessageLike", () => {
+  it("preserves modality on user and assistant messages", () => {
+    const user = fromThreadMessageLike(
+      {
+        role: "user",
+        content: "Hello",
+        metadata: { modality: "voice" },
+      },
+      "user-id",
+      { type: "complete", reason: "unknown" },
+    );
+    const assistant = fromThreadMessageLike(
+      {
+        role: "assistant",
+        content: "Hi",
+        metadata: { modality: "voice" },
+      },
+      "assistant-id",
+      { type: "complete", reason: "unknown" },
+    );
+
+    expect(user.metadata.modality).toBe("voice");
+    expect(assistant.metadata.modality).toBe("voice");
+  });
+
+  it("leaves modality absent when the input has none", () => {
+    const message = fromThreadMessageLike(
+      { role: "user", content: "Hello" },
+      "user-id",
+      { type: "complete", reason: "unknown" },
+    );
+
+    expect(message.metadata).not.toHaveProperty("modality");
+  });
+
+  it("ignores modality on system messages", () => {
+    const message = fromThreadMessageLike(
+      {
+        role: "system",
+        content: "Instructions",
+        metadata: { modality: "voice" },
+      },
+      "system-id",
+      { type: "complete", reason: "unknown" },
+    );
+
+    expect(message.metadata).not.toHaveProperty("modality");
+  });
+});

package/src/runtime/utils/thread-message-like.ts

@@ -20,6 +20,7 @@ import type {
 } from "../../types/message";
 import type { CompleteAttachment } from "../../types/attachment";
 import type {
+  MessageModality,
   MessageTiming,
   PartProviderMetadata,
   TextMessagePart,
@@ -88,6 +89,7 @@ export type ThreadMessageLike = {
         readonly timing?: MessageTiming | undefined;
         readonly submittedFeedback?: { readonly type: "positive" | "negative" };
         readonly isOptimistic?: boolean | undefined;
+        readonly modality?: MessageModality | undefined;
         readonly custom?: Record<string, unknown> | undefined;
       }
     | undefined;
@@ -223,6 +225,7 @@ export const fromThreadMessageLike = (
             submittedFeedback: metadata.submittedFeedback,
           }),
           ...(metadata?.isOptimistic && { isOptimistic: true }),
+          ...(metadata?.modality && { modality: metadata.modality }),
         },
       } satisfies ThreadAssistantMessage;
 
@@ -263,6 +266,7 @@ export const fromThreadMessageLike = (
         metadata: {
           custom: metadata?.custom ?? {},
           ...(metadata?.isOptimistic && { isOptimistic: true }),
+          ...(metadata?.modality && { modality: metadata.modality }),
         },
       } satisfies ThreadUserMessage;
 

package/src/runtimes/external-store/external-store-thread-runtime-core.ts

@@ -637,6 +637,8 @@ export class ExternalStoreThreadRuntimeCore
   }
 
   public async append(rawMessage: AppendMessage): Promise<void> {
+    if (this._isVoiceMessage(rawMessage.sourceId))
+      throw new Error("Voice transcript messages cannot be edited");
     // sourceId marks an edit send; the parent may coincide with the head
     // after a resync (e.g. cancelRun dropped the edited message).
     const isEdit =
@@ -786,6 +788,8 @@ export class ExternalStoreThreadRuntimeCore
   public async startRun(config: StartRunConfig): Promise<void> {
     if (!this._store.onReload)
       throw new Error("Runtime does not support reloading messages.");
+    if (this._isVoiceMessage(config.sourceId))
+      throw new Error("Voice transcript messages cannot be reloaded");
 
     this._pendingDeleteEvictions.clear();
 
@@ -800,6 +804,8 @@ export class ExternalStoreThreadRuntimeCore
   public async resumeRun(config: ResumeRunConfig): Promise<void> {
     if (!this._store.onResume)
       throw new Error("Runtime does not support resuming runs.");
+    if (this._isVoiceMessage(config.sourceId))
+      throw new Error("Voice transcript messages cannot be reloaded");
 
     await this._store.onResume(config);
   }
@@ -961,8 +967,32 @@ export class ExternalStoreThreadRuntimeCore
   ): Promise<void> {
     if (!this._store.onRespondToToolApproval)
       throw new Error("Runtime does not support tool approvals.");
+    const message = this.messages.findLast(
+      (candidate) =>
+        candidate.role === "assistant" &&
+        candidate.content.some(
+          (part) =>
+            part.type === "tool-call" &&
+            part.approval?.id === options.approvalId,
+        ),
+    );
+    const toolCall = message?.content.find(
+      (part) =>
+        part.type === "tool-call" && part.approval?.id === options.approvalId,
+    );
     try {
-      return Promise.resolve(this._store.onRespondToToolApproval(options));
+      return Promise.resolve(this._store.onRespondToToolApproval(options)).then(
+        () => {
+          if (message && toolCall?.type === "tool-call") {
+            this._notifyToolApprovalAnswered(
+              message.id,
+              toolCall.toolCallId,
+              toolCall.toolName,
+              options.approved,
+            );
+          }
+        },
+      );
     } catch (error) {
       return Promise.reject(error);
     }