@assistant-ui/core 0.3.18 → 0.3.19

package/src/runtime/base/base-thread-runtime-core.ts

@@ -36,7 +36,10 @@ import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 import { notifyEventListeners } from "../../utils/notify-event-listeners";
 import { gateInteractableComposerMetadata } from "../../model-context/interactable-composer-metadata";
-import { BaseSubscribable } from "../../subscribable/subscribable";
+import {
+  BaseSubscribable,
+  notifySubscribers,
+} from "../../subscribable/subscribable";
 
 type BaseThreadAdapters = {
   speech?: SpeechSynthesisAdapter | undefined;
@@ -168,7 +171,16 @@ export abstract class BaseThreadRuntimeCore
   public getEditComposer(messageId: string) {
     return this._editComposers.get(messageId);
   }
+  protected _isVoiceMessage(messageId: string | null) {
+    return (
+      messageId !== null && this._voiceMessages.some((m) => m.id === messageId)
+    );
+  }
+
   public beginEdit(messageId: string) {
+    if (this._isVoiceMessage(messageId)) {
+      throw new Error("Voice transcript messages cannot be edited");
+    }
     if (this._editComposers.has(messageId))
       throw new Error("Edit already in progress");
 
@@ -227,11 +239,27 @@ export abstract class BaseThreadRuntimeCore
     notifyEventListeners(subscribers, payload, `Thread runtime "${event}"`);
   }
 
+  protected _notifyToolApprovalAnswered(
+    messageId: string,
+    toolCallId: string,
+    toolName: string,
+    approved: boolean,
+  ) {
+    this._notifyEventSubscribers("toolApprovalAnswered", {
+      messageId,
+      toolCallId,
+      toolName,
+      approved,
+    });
+  }
+
   public submitFeedback({ messageId, type }: SubmitFeedbackOptions) {
     const adapter = this.adapters?.feedback;
     if (!adapter) throw new Error("Feedback adapter not configured");
 
-    const { message, parentId } = this.repository.getMessage(messageId);
+    const entry = this.getMessageById(messageId);
+    if (!entry) throw new Error(`Message not found: ${messageId}`);
+    const { message, parentId } = entry;
     adapter.submit({ message, type });
 
     if (message.role === "assistant") {
@@ -242,7 +270,18 @@ export abstract class BaseThreadRuntimeCore
           submittedFeedback: { type },
         },
       };
-      this.repository.addOrUpdateMessage(parentId, updatedMessage);
+      const voiceIdx = this._voiceMessages.findIndex(
+        (voiceMessage) => voiceMessage.id === messageId,
+      );
+      if (voiceIdx === -1) {
+        this.repository.addOrUpdateMessage(parentId, updatedMessage);
+      } else {
+        this._voiceMessages[voiceIdx] = updatedMessage;
+        if (this._currentAssistantMsg === message) {
+          this._currentAssistantMsg = updatedMessage as ThreadAssistantMessage;
+        }
+        this._markVoiceMessagesDirty();
+      }
     }
 
     this._notifySubscribers();
@@ -255,36 +294,80 @@ export abstract class BaseThreadRuntimeCore
     const adapter = this.adapters?.speech;
     if (!adapter) throw new Error("Speech adapter not configured");
 
-    const { message } = this.repository.getMessage(messageId);
-
-    this._stopSpeaking?.();
+    const entry = this.getMessageById(messageId);
+    if (!entry) throw new Error(`Message not found: ${messageId}`);
+    const { message } = entry;
 
-    const utterance = adapter.speak(getThreadMessageText(message));
-    const unsub = utterance.subscribe(() => {
+    const previousStop = this._stopSpeaking;
+    let utterance: SpeechSynthesisAdapter.Utterance;
+    try {
+      previousStop?.();
+      utterance = adapter.speak(getThreadMessageText(message));
+    } catch (error) {
+      if (previousStop && !this._stopSpeaking) {
+        try {
+          this._notifySubscribers();
+        } catch (notificationError) {
+          console.error(
+            "[assistant-ui] Speech rollback notification threw",
+            notificationError,
+          );
+        }
+      }
+      throw error;
+    }
+    let unsub: Unsubscribe | undefined;
+    const clear = () => {
+      this._stopSpeaking = undefined;
+      this.speech = undefined;
+      const cleanup = unsub;
+      unsub = undefined;
+      cleanup?.();
+    };
+    const stop = () => {
+      if (this._stopSpeaking !== stop) return;
+      try {
+        clear();
+      } finally {
+        utterance.cancel();
+      }
+    };
+    const update = () => {
+      if (this._stopSpeaking !== stop) return;
       if (utterance.status.type === "ended") {
-        this._stopSpeaking = undefined;
-        this.speech = undefined;
+        notifySubscribers([clear, () => this._notifySubscribers()]);
       } else {
         this.speech = { messageId, status: utterance.status };
+        this._notifySubscribers();
       }
-      this._notifySubscribers();
-    });
-
-    this.speech = { messageId, status: utterance.status };
-    this._notifySubscribers();
-
-    this._stopSpeaking = () => {
-      utterance.cancel();
-      unsub();
-      this.speech = undefined;
-      this._stopSpeaking = undefined;
     };
+
+    this._stopSpeaking = stop;
+    try {
+      unsub = utterance.subscribe(update);
+      if (this._stopSpeaking !== stop) {
+        unsub();
+        return;
+      }
+      update();
+    } catch (error) {
+      if (this._stopSpeaking === stop) {
+        try {
+          notifySubscribers([stop, () => this._notifySubscribers()]);
+        } catch (cleanupError) {
+          console.error(
+            "[assistant-ui] Speech rollback cleanup threw",
+            cleanupError,
+          );
+        }
+      }
+      throw error;
+    }
   }
 
   public stopSpeaking() {
     if (!this._stopSpeaking) throw new Error("No message is being spoken");
-    this._stopSpeaking();
-    this._notifySubscribers();
+    notifySubscribers([this._stopSpeaking, () => this._notifySubscribers()]);
   }
 
   private _voiceSession: RealtimeVoiceAdapter.Session | undefined;
@@ -305,67 +388,111 @@ export abstract class BaseThreadRuntimeCore
     const adapter = this.adapters?.voice;
     if (!adapter) throw new Error("Voice adapter not configured");
 
-    this.disconnectVoice();
+    try {
+      this.disconnectVoice();
+    } catch (error) {
+      console.error(
+        "[assistant-ui] Voice cleanup threw before reconnect",
+        error,
+      );
+    }
 
     const session = adapter.connect({});
     this._voiceSession = session;
     const unsubs: Array<() => void> = [];
+    this._voiceUnsubs = unsubs;
 
-    let currentMode: RealtimeVoiceAdapter.Mode = "listening";
+    // The cleanup-list identity preserves ownership after an ended status clears the session.
+    const finishDetachedSetup = () => {
+      if (this._voiceSession === session && this._voiceUnsubs === unsubs) {
+        return false;
+      }
 
-    this.voice = {
-      status: session.status,
-      isMuted: session.isMuted,
-      mode: currentMode,
+      try {
+        notifySubscribers(unsubs.splice(0));
+      } catch (error) {
+        console.error(
+          "[assistant-ui] Detached voice setup cleanup threw",
+          error,
+        );
+      }
+      return true;
     };
-    this._voiceVolume = 0;
-    this._notifySubscribers();
 
-    unsubs.push(
-      session.onStatusChange((status) => {
-        if (status.type === "ended") {
-          this._finishVoiceAssistantMessage();
-          this._voiceSession = undefined;
-          this.voice = undefined;
-        } else {
-          this.voice = {
-            status,
-            isMuted: session.isMuted,
-            mode: currentMode,
-          };
-        }
-        this._notifySubscribers();
-      }),
-    );
+    try {
+      let currentMode: RealtimeVoiceAdapter.Mode = "listening";
 
-    unsubs.push(
-      session.onModeChange((mode) => {
-        currentMode = mode;
-        if (this.voice) {
-          this.voice = { ...this.voice, mode };
+      this.voice = {
+        status: session.status,
+        isMuted: session.isMuted,
+        mode: currentMode,
+      };
+      this._voiceVolume = 0;
+      this._notifySubscribers();
+      if (finishDetachedSetup()) return;
+
+      unsubs.push(
+        session.onStatusChange((status) => {
+          if (status.type === "ended") {
+            this._finishVoiceAssistantMessage();
+            this._voiceSession = undefined;
+            this.voice = undefined;
+          } else {
+            this.voice = {
+              status,
+              isMuted: session.isMuted,
+              mode: currentMode,
+            };
+          }
           this._notifySubscribers();
-        }
-      }),
-    );
-
-    unsubs.push(
-      session.onVolumeChange((volume) => {
-        this._voiceVolume = volume;
-        notifyEventListeners(
-          this._voiceVolumeSubscribers,
-          undefined,
-          "Voice volume",
-        );
-      }),
-    );
-
-    unsubs.push(
-      session.onTranscript((transcript) => {
-        this._handleVoiceTranscript(transcript);
-      }),
-    );
+        }),
+      );
+      if (finishDetachedSetup()) return;
+
+      unsubs.push(
+        session.onModeChange((mode) => {
+          currentMode = mode;
+          if (this.voice) {
+            this.voice = { ...this.voice, mode };
+            this._notifySubscribers();
+          }
+        }),
+      );
+      if (finishDetachedSetup()) return;
+
+      unsubs.push(
+        session.onVolumeChange((volume) => {
+          this._voiceVolume = volume;
+          notifyEventListeners(
+            this._voiceVolumeSubscribers,
+            undefined,
+            "Voice volume",
+          );
+        }),
+      );
+      if (finishDetachedSetup()) return;
 
-    this._voiceUnsubs = unsubs;
+      unsubs.push(
+        session.onTranscript((transcript) => {
+          this._handleVoiceTranscript(transcript);
+        }),
+      );
+      finishDetachedSetup();
+    } catch (error) {
+      if (this._voiceSession === session && this._voiceUnsubs === unsubs) {
+        try {
+          this.disconnectVoice();
+        } catch (cleanupError) {
+          console.error(
+            "[assistant-ui] Voice rollback cleanup threw",
+            cleanupError,
+          );
+        }
+      } else {
+        finishDetachedSetup();
+      }
+      throw error;
+    }
   }
 
   private _currentAssistantMsg: ThreadAssistantMessage | null = null;
@@ -384,7 +511,7 @@ export abstract class BaseThreadRuntimeCore
           id: generateId(),
           role: "user",
           content: [{ type: "text", text: transcript.text }],
-          metadata: { custom: {} },
+          metadata: { modality: "voice", custom: {} },
           createdAt: new Date(),
           status: { type: "complete", reason: "unknown" },
           attachments: [],
@@ -407,6 +534,7 @@ export abstract class BaseThreadRuntimeCore
             unstable_annotations: [],
             unstable_data: [],
             steps: [],
+            modality: "voice",
             custom: {},
           },
           status,
@@ -448,22 +576,33 @@ export abstract class BaseThreadRuntimeCore
   }
 
   public disconnectVoice() {
-    this._finishVoiceAssistantMessage();
     this._currentAssistantMsg = null;
-    for (const unsub of this._voiceUnsubs) unsub();
+    // Drain the shared list in place so reentrant setup cannot release the same handles again.
+    const unsubs = this._voiceUnsubs.splice(0);
     this._voiceUnsubs = [];
-    this._voiceSession?.disconnect();
+    const session = this._voiceSession;
     this._voiceSession = undefined;
     this.voice = undefined;
     this._voiceVolume = 0;
-    notifyEventListeners(
-      this._voiceVolumeSubscribers,
-      undefined,
-      "Voice volume",
-    );
+    const stopSpeaking =
+      this.speech && this._isVoiceMessage(this.speech.messageId)
+        ? this._stopSpeaking
+        : undefined;
     this._voiceMessages = [];
     this._markVoiceMessagesDirty();
-    this._notifySubscribers();
+
+    notifySubscribers([
+      ...unsubs,
+      ...(stopSpeaking ? [stopSpeaking] : []),
+      ...(session ? [() => session.disconnect()] : []),
+      () =>
+        notifyEventListeners(
+          this._voiceVolumeSubscribers,
+          undefined,
+          "Voice volume",
+        ),
+      () => this._notifySubscribers(),
+    ]);
   }
 
   public muteVoice() {

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -98,6 +98,12 @@ export type SubmittedFeedback = {
 };
 
 export type ThreadRuntimeEventPayload = {
+  toolApprovalAnswered: {
+    messageId: string;
+    toolCallId: string;
+    toolName: string;
+    approved: boolean;
+  };
   /**
    * @deprecated State-derivable. Observe `state.isRunning` flipping to `true`
    * via `subscribe` + `getState` instead. Note: this event fires at the

package/src/runtime/queue/message-queue.ts

@@ -13,6 +13,11 @@ import type {
 } from "./external-thread-queue-adapter";
 
 export type MessageQueueDriver = {
+  /**
+   * A synchronous throw is treated as a run that never started and restores
+   * the message. A driver that already started work must call `notifyBusy`
+   * before throwing.
+   */
   run: (message: AppendMessage, options: { steer: boolean }) => void;
   /** When omitted, steering degrades to "process next" instead of interrupting. */
   cancel?: (() => void) | undefined;
@@ -38,6 +43,12 @@ export type MessageQueueController = {
 
 type Lane = "queue" | "steer";
 
+type DispatchItem = {
+  id: string;
+  item: QueueItemState;
+  message: AppendMessage;
+};
+
 const getQueueItemParts = (
   message: AppendMessage,
 ): readonly (FileMessagePart | TextMessagePart)[] => {
@@ -83,6 +94,7 @@ export const createMessageQueue = (
   // settles from cancelled runs that must drop `running` without advancing
   let cancelSettles = 0;
   let interrupting = false;
+  let busyEdges = 0;
 
   const notify = () => {
     notifyEventListeners(subscribers, undefined, "Message queue");
@@ -101,6 +113,19 @@ export const createMessageQueue = (
     parts: getQueueItemParts(message),
   });
 
+  const restore = (lane: Lane, dispatch: DispatchItem, index = 0) => {
+    paused = true;
+    messages.set(dispatch.id, dispatch.message);
+    setLanes({
+      ...lanes,
+      [lane]: [
+        ...lanes[lane].slice(0, index),
+        dispatch.item,
+        ...lanes[lane].slice(index),
+      ],
+    });
+  };
+
   const laneOf = (queueItemId: string): Lane | undefined => {
     if (lanes.steer.some((item) => item.id === queueItemId)) return "steer";
     if (lanes.queue.some((item) => item.id === queueItemId)) return "queue";
@@ -116,27 +141,61 @@ export const createMessageQueue = (
     messages.delete(head.id);
     setLanes({ ...lanes, [lane]: lanes[lane].slice(1) });
     if (!message) return;
+    const dispatch = { id: head.id, item: head, message };
     running = true;
-    driver.run(dispatchTransform(message), { steer: false });
+    const busyEdgesBeforeRun = busyEdges;
+    try {
+      driver.run(dispatchTransform(message), { steer: false });
+    } catch (error) {
+      if (busyEdges === busyEdgesBeforeRun) {
+        running = false;
+        restore(lane, dispatch);
+      }
+      throw error;
+    }
   };
 
-  const interrupt = (message: AppendMessage) => {
+  const interrupt = (
+    dispatch: DispatchItem,
+    restoreLane: Lane = "steer",
+    restoreIndex = 0,
+  ) => {
     paused = false;
     // the interrupted run settles exactly once, whether or not it was
     // already cancel-notified
     suppressIdle += Math.max(cancelSettles, 1);
     cancelSettles = 0;
+    const restoreInterrupted = (replacementStarted = false) => {
+      if (replacementStarted) return;
+      // The live count distinguishes an outstanding cancellation settle
+      // from one delivered synchronously by cancel().
+      const pendingSettles = suppressIdle;
+      suppressIdle = Math.max(pendingSettles - 1, 0);
+      cancelSettles = pendingSettles > 0 ? 1 : 0;
+      running = pendingSettles > 0;
+      restore(restoreLane, dispatch, restoreIndex);
+    };
     // a driver whose cancel routes through the runtime notifies this queue
     // back; the interrupt already accounted for that settle and is dispatching
     // in its place
     interrupting = true;
     try {
       driver.cancel!();
+    } catch (error) {
+      restoreInterrupted();
+      throw error;
     } finally {
       interrupting = false;
     }
     running = true;
-    driver.run(dispatchTransform(message), { steer: true });
+    const busyEdgesBeforeRun = busyEdges;
+    try {
+      driver.run(dispatchTransform(dispatch.message), { steer: true });
+    } catch (error) {
+      const replacementStarted = busyEdges !== busyEdgesBeforeRun;
+      restoreInterrupted(replacementStarted);
+      throw error;
+    }
   };
 
   const push = (lane: Lane, message: AppendMessage) => {
@@ -153,7 +212,8 @@ export const createMessageQueue = (
 
   const steer = (message: AppendMessage) => {
     if (running && driver.cancel) {
-      interrupt(message);
+      const id = generateId();
+      interrupt({ id, item: toItem(id, message), message });
       return;
     }
     push("steer", message);
@@ -164,7 +224,8 @@ export const createMessageQueue = (
     if (!fromLane) throw new Error(`Unknown queue item "${queueItemId}".`);
     const toLane = placement.lane ?? fromLane;
 
-    const item = lanes[fromLane].find((i) => i.id === queueItemId)!;
+    const fromIndex = lanes[fromLane].findIndex((i) => i.id === queueItemId);
+    const item = lanes[fromLane][fromIndex]!;
     const dest = (toLane === fromLane ? lanes[fromLane] : lanes[toLane]).filter(
       (i) => i.id !== queueItemId,
     );
@@ -217,7 +278,7 @@ export const createMessageQueue = (
         queue: lanes.queue.filter((i) => i.id !== queueItemId),
         steer: lanes.steer,
       });
-      interrupt(message);
+      interrupt({ id: queueItemId, item, message }, fromLane, fromIndex);
       return;
     }
 
@@ -280,6 +341,7 @@ export const createMessageQueue = (
       suppressIdle += cancelSettles;
       cancelSettles = 0;
       running = true;
+      busyEdges++;
     },
     notifyIdle: () => {
       if (suppressIdle > 0) {

package/src/runtime/utils/auto-status.test.ts

@@ -1,5 +1,33 @@
 import { describe, expect, it } from "vitest";
-import { getAutoStatus } from "./auto-status";
+import type { MessageStatus, ThreadMessage } from "../../types/message";
+import {
+  getAutoStatus,
+  getContentAutoStatus,
+  isBackgroundToolCall,
+} from "./auto-status";
+
+const nestedAssistantMessage = (status: MessageStatus): ThreadMessage => ({
+  id: "nested-assistant",
+  createdAt: new Date(0),
+  role: "assistant",
+  content: [],
+  status,
+  metadata: {
+    unstable_state: {},
+    unstable_annotations: [],
+    unstable_data: [],
+    steps: [],
+    custom: {},
+  },
+});
+
+const pendingToolCall = (messages?: readonly ThreadMessage[]) => ({
+  type: "tool-call" as const,
+  toolName: "delegate",
+  args: {},
+  argsText: "",
+  messages,
+});
 
 describe("getAutoStatus", () => {
   it("reports a cancelled message as incomplete", () => {
@@ -54,4 +82,87 @@ describe("getAutoStatus", () => {
       ).toMatchObject(expected);
     },
   );
+
+  it.each([true, false])(
+    "keeps a background tool call running when isLast is %s",
+    (isLast) => {
+      expect(
+        getContentAutoStatus(
+          [pendingToolCall([nestedAssistantMessage({ type: "running" })])],
+          isLast,
+          false,
+        ),
+      ).toMatchObject({ type: "running" });
+    },
+  );
+
+  it("reads the trailing nested message, not an earlier running one", () => {
+    expect(
+      isBackgroundToolCall(
+        pendingToolCall([
+          nestedAssistantMessage({ type: "running" }),
+          nestedAssistantMessage({ type: "complete", reason: "unknown" }),
+        ]),
+      ),
+    ).toBe(false);
+  });
+
+  it("lets cancellation clear a background tool call", () => {
+    expect(
+      getAutoStatus(true, false, false, true, undefined, true, true),
+    ).toMatchObject({ type: "requires-action", reason: "tool-calls" });
+  });
+
+  it("keeps a settled nested tool call pending", () => {
+    expect(
+      getContentAutoStatus(
+        [
+          pendingToolCall([
+            nestedAssistantMessage({ type: "complete", reason: "unknown" }),
+          ]),
+        ],
+        true,
+        false,
+      ),
+    ).toMatchObject({ type: "requires-action", reason: "tool-calls" });
+  });
+
+  it("keeps an interrupted tool call ahead of a background tool call", () => {
+    expect(
+      getContentAutoStatus(
+        [
+          {
+            ...pendingToolCall(),
+            interrupt: { type: "human", payload: {} },
+          },
+          pendingToolCall([nestedAssistantMessage({ type: "running" })]),
+        ],
+        false,
+        false,
+      ),
+    ).toMatchObject({ type: "requires-action", reason: "interrupt" });
+  });
+
+  it.each([
+    ["a result", { ...pendingToolCall(), result: {} }],
+    ["no nested messages", pendingToolCall()],
+    [
+      "only a user nested message",
+      pendingToolCall([
+        {
+          id: "nested-user",
+          createdAt: new Date(0),
+          role: "user",
+          content: [],
+          attachments: [],
+          metadata: { custom: {} },
+        },
+      ]),
+    ],
+  ])(
+    "does not treat a tool call with %s as a background call",
+    (_label, call) => {
+      expect(isBackgroundToolCall(call)).toBe(false);
+    },
+  );
 });