@assistant-ui/core 0.3.18 → 0.3.20

package/src/runtime/base/base-thread-runtime-core.ts

@@ -36,7 +36,10 @@ import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 import { notifyEventListeners } from "../../utils/notify-event-listeners";
 import { gateInteractableComposerMetadata } from "../../model-context/interactable-composer-metadata";
-import { BaseSubscribable } from "../../subscribable/subscribable";
+import {
+  BaseSubscribable,
+  notifySubscribers,
+} from "../../subscribable/subscribable";
 
 type BaseThreadAdapters = {
   speech?: SpeechSynthesisAdapter | undefined;
@@ -89,6 +92,8 @@ export abstract class BaseThreadRuntimeCore
     return this.repository.getMessages();
   }
 
+  protected _commitVoiceMessage(_message: ThreadMessage): void {}
+
   public get messages(): readonly ThreadMessage[] {
     if (this._voiceMessages.length === 0) {
       return this._getBaseMessages();
@@ -98,7 +103,13 @@ export abstract class BaseThreadRuntimeCore
       this._cachedVoiceGeneration !== this._voiceGeneration ||
       this._cachedMergedBase !== base
     ) {
-      this._cachedMergedMessages = [...base, ...this._voiceMessages];
+      const baseMessageIds = new Set(base.map((message) => message.id));
+      this._cachedMergedMessages = [
+        ...base,
+        ...this._voiceMessages.filter(
+          (message) => !baseMessageIds.has(message.id),
+        ),
+      ];
       this._cachedVoiceGeneration = this._voiceGeneration;
       this._cachedMergedBase = base;
     }
@@ -168,7 +179,26 @@ export abstract class BaseThreadRuntimeCore
   public getEditComposer(messageId: string) {
     return this._editComposers.get(messageId);
   }
+  protected _isVoiceMessage(messageId: string | null) {
+    return (
+      messageId !== null && this._voiceMessages.some((m) => m.id === messageId)
+    );
+  }
+
+  protected _resolveAppendParent(parentId: string | null): string | null {
+    return this._isVoiceMessage(parentId)
+      ? (this._getBaseMessages().at(-1)?.id ?? null)
+      : parentId;
+  }
+
   public beginEdit(messageId: string) {
+    if (this.voice)
+      throw new Error(
+        "Cannot edit a message while a voice session is connected",
+      );
+    if (this._isVoiceMessage(messageId)) {
+      throw new Error("Voice transcript messages cannot be edited");
+    }
     if (this._editComposers.has(messageId))
       throw new Error("Edit already in progress");
 
@@ -227,22 +257,49 @@ export abstract class BaseThreadRuntimeCore
     notifyEventListeners(subscribers, payload, `Thread runtime "${event}"`);
   }
 
-  public submitFeedback({ messageId, type }: SubmitFeedbackOptions) {
-    const adapter = this.adapters?.feedback;
-    if (!adapter) throw new Error("Feedback adapter not configured");
+  protected _notifyToolApprovalAnswered(
+    messageId: string,
+    toolCallId: string,
+    toolName: string,
+    approved: boolean,
+  ) {
+    this._notifyEventSubscribers("toolApprovalAnswered", {
+      messageId,
+      toolCallId,
+      toolName,
+      approved,
+    });
+  }
 
-    const { message, parentId } = this.repository.getMessage(messageId);
-    adapter.submit({ message, type });
+  public submitFeedback({ messageId, type, comment }: SubmitFeedbackOptions) {
+    const adapter = this.adapters?.feedback;
+    const entry = this.getMessageById(messageId);
+    if (!entry) throw new Error(`Message not found: ${messageId}`);
+    const { message, parentId } = entry;
+    const trimmed = comment?.trim();
+    const feedback = { type, ...(trimmed ? { comment: trimmed } : undefined) };
+    adapter?.submit({ message, ...feedback });
 
     if (message.role === "assistant") {
       const updatedMessage: ThreadMessage = {
         ...message,
         metadata: {
           ...message.metadata,
-          submittedFeedback: { type },
+          submittedFeedback: feedback,
         },
       };
-      this.repository.addOrUpdateMessage(parentId, updatedMessage);
+      const voiceIdx = this._voiceMessages.findIndex(
+        (voiceMessage) => voiceMessage.id === messageId,
+      );
+      if (voiceIdx === -1) {
+        this.repository.addOrUpdateMessage(parentId, updatedMessage);
+      } else {
+        this._voiceMessages[voiceIdx] = updatedMessage;
+        if (this._currentAssistantMsg === message) {
+          this._currentAssistantMsg = updatedMessage as ThreadAssistantMessage;
+        }
+        this._markVoiceMessagesDirty();
+      }
     }
 
     this._notifySubscribers();
@@ -255,36 +312,80 @@ export abstract class BaseThreadRuntimeCore
     const adapter = this.adapters?.speech;
     if (!adapter) throw new Error("Speech adapter not configured");
 
-    const { message } = this.repository.getMessage(messageId);
-
-    this._stopSpeaking?.();
+    const entry = this.getMessageById(messageId);
+    if (!entry) throw new Error(`Message not found: ${messageId}`);
+    const { message } = entry;
 
-    const utterance = adapter.speak(getThreadMessageText(message));
-    const unsub = utterance.subscribe(() => {
+    const previousStop = this._stopSpeaking;
+    let utterance: SpeechSynthesisAdapter.Utterance;
+    try {
+      previousStop?.();
+      utterance = adapter.speak(getThreadMessageText(message));
+    } catch (error) {
+      if (previousStop && !this._stopSpeaking) {
+        try {
+          this._notifySubscribers();
+        } catch (notificationError) {
+          console.error(
+            "[assistant-ui] Speech rollback notification threw",
+            notificationError,
+          );
+        }
+      }
+      throw error;
+    }
+    let unsub: Unsubscribe | undefined;
+    const clear = () => {
+      this._stopSpeaking = undefined;
+      this.speech = undefined;
+      const cleanup = unsub;
+      unsub = undefined;
+      cleanup?.();
+    };
+    const stop = () => {
+      if (this._stopSpeaking !== stop) return;
+      try {
+        clear();
+      } finally {
+        utterance.cancel();
+      }
+    };
+    const update = () => {
+      if (this._stopSpeaking !== stop) return;
       if (utterance.status.type === "ended") {
-        this._stopSpeaking = undefined;
-        this.speech = undefined;
+        notifySubscribers([clear, () => this._notifySubscribers()]);
       } else {
         this.speech = { messageId, status: utterance.status };
+        this._notifySubscribers();
       }
-      this._notifySubscribers();
-    });
-
-    this.speech = { messageId, status: utterance.status };
-    this._notifySubscribers();
-
-    this._stopSpeaking = () => {
-      utterance.cancel();
-      unsub();
-      this.speech = undefined;
-      this._stopSpeaking = undefined;
     };
+
+    this._stopSpeaking = stop;
+    try {
+      unsub = utterance.subscribe(update);
+      if (this._stopSpeaking !== stop) {
+        unsub();
+        return;
+      }
+      update();
+    } catch (error) {
+      if (this._stopSpeaking === stop) {
+        try {
+          notifySubscribers([stop, () => this._notifySubscribers()]);
+        } catch (cleanupError) {
+          console.error(
+            "[assistant-ui] Speech rollback cleanup threw",
+            cleanupError,
+          );
+        }
+      }
+      throw error;
+    }
   }
 
   public stopSpeaking() {
     if (!this._stopSpeaking) throw new Error("No message is being spoken");
-    this._stopSpeaking();
-    this._notifySubscribers();
+    notifySubscribers([this._stopSpeaking, () => this._notifySubscribers()]);
   }
 
   private _voiceSession: RealtimeVoiceAdapter.Session | undefined;
@@ -301,71 +402,145 @@ export abstract class BaseThreadRuntimeCore
     return () => this._voiceVolumeSubscribers.delete(callback);
   };
 
+  protected _onVoiceConnected(): void {}
+
+  protected _onVoiceDisconnected(): void {}
+
+  protected _isRunActive(): boolean {
+    const runtime: ThreadRuntimeCore = this;
+    if (runtime.isRunning) return true;
+    const last = this._getBaseMessages().at(-1);
+    return (
+      last?.role === "assistant" &&
+      (last.status.type === "running" || last.status.type === "requires-action")
+    );
+  }
+
   public connectVoice() {
     const adapter = this.adapters?.voice;
     if (!adapter) throw new Error("Voice adapter not configured");
+    if (this._isRunActive())
+      throw new Error(
+        "Cannot start a voice session while a run is in progress or paused on a pending tool action",
+      );
+    const replacing = this._voiceSession !== undefined;
 
-    this.disconnectVoice();
+    try {
+      this._disconnectVoice(false);
+    } catch (error) {
+      console.error(
+        "[assistant-ui] Voice cleanup threw before reconnect",
+        error,
+      );
+    }
 
-    const session = adapter.connect({});
+    let session: RealtimeVoiceAdapter.Session;
+    try {
+      session = adapter.connect({});
+    } catch (error) {
+      if (replacing && this._voiceSession === undefined)
+        this._onVoiceDisconnected();
+      throw error;
+    }
     this._voiceSession = session;
     const unsubs: Array<() => void> = [];
+    this._voiceUnsubs = unsubs;
 
-    let currentMode: RealtimeVoiceAdapter.Mode = "listening";
+    // The cleanup-list identity preserves ownership after an ended status clears the session.
+    const finishDetachedSetup = () => {
+      if (this._voiceSession === session && this._voiceUnsubs === unsubs) {
+        return false;
+      }
 
-    this.voice = {
-      status: session.status,
-      isMuted: session.isMuted,
-      mode: currentMode,
+      try {
+        notifySubscribers(unsubs.splice(0));
+      } catch (error) {
+        console.error(
+          "[assistant-ui] Detached voice setup cleanup threw",
+          error,
+        );
+      }
+      return true;
     };
-    this._voiceVolume = 0;
-    this._notifySubscribers();
 
-    unsubs.push(
-      session.onStatusChange((status) => {
-        if (status.type === "ended") {
-          this._finishVoiceAssistantMessage();
-          this._voiceSession = undefined;
-          this.voice = undefined;
-        } else {
-          this.voice = {
-            status,
-            isMuted: session.isMuted,
-            mode: currentMode,
-          };
-        }
-        this._notifySubscribers();
-      }),
-    );
+    try {
+      let currentMode: RealtimeVoiceAdapter.Mode = "listening";
 
-    unsubs.push(
-      session.onModeChange((mode) => {
-        currentMode = mode;
-        if (this.voice) {
-          this.voice = { ...this.voice, mode };
+      this.voice = {
+        status: session.status,
+        isMuted: session.isMuted,
+        mode: currentMode,
+      };
+      this._voiceVolume = 0;
+      this._notifySubscribers();
+      if (finishDetachedSetup()) return;
+
+      unsubs.push(
+        session.onStatusChange((status) => {
+          if (this._voiceSession !== session) return;
+          if (status.type === "ended") {
+            this._finishVoiceAssistantMessage();
+            this._voiceSession = undefined;
+            this.voice = undefined;
+            this._onVoiceDisconnected();
+          } else {
+            this.voice = {
+              status,
+              isMuted: session.isMuted,
+              mode: currentMode,
+            };
+          }
           this._notifySubscribers();
-        }
-      }),
-    );
-
-    unsubs.push(
-      session.onVolumeChange((volume) => {
-        this._voiceVolume = volume;
-        notifyEventListeners(
-          this._voiceVolumeSubscribers,
-          undefined,
-          "Voice volume",
-        );
-      }),
-    );
-
-    unsubs.push(
-      session.onTranscript((transcript) => {
-        this._handleVoiceTranscript(transcript);
-      }),
-    );
+        }),
+      );
+      if (finishDetachedSetup()) return;
+
+      unsubs.push(
+        session.onModeChange((mode) => {
+          currentMode = mode;
+          if (this.voice) {
+            this.voice = { ...this.voice, mode };
+            this._notifySubscribers();
+          }
+        }),
+      );
+      if (finishDetachedSetup()) return;
+
+      unsubs.push(
+        session.onVolumeChange((volume) => {
+          this._voiceVolume = volume;
+          notifyEventListeners(
+            this._voiceVolumeSubscribers,
+            undefined,
+            "Voice volume",
+          );
+        }),
+      );
+      if (finishDetachedSetup()) return;
 
-    this._voiceUnsubs = unsubs;
+      unsubs.push(
+        session.onTranscript((transcript) => {
+          this._handleVoiceTranscript(transcript);
+        }),
+      );
+      if (!finishDetachedSetup()) this._onVoiceConnected();
+    } catch (error) {
+      if (this._voiceSession === session && this._voiceUnsubs === unsubs) {
+        try {
+          this._disconnectVoice(false);
+        } catch (cleanupError) {
+          console.error(
+            "[assistant-ui] Voice rollback cleanup threw",
+            cleanupError,
+          );
+        }
+        if (replacing && this._voiceSession === undefined)
+          this._onVoiceDisconnected();
+      } else {
+        finishDetachedSetup();
+      }
+      throw error;
+    }
   }
 
   private _currentAssistantMsg: ThreadAssistantMessage | null = null;
@@ -380,15 +555,17 @@ export abstract class BaseThreadRuntimeCore
       this._currentAssistantMsg = null;
 
       if (transcript.isFinal) {
-        this._voiceMessages.push({
+        const message: ThreadMessage = {
           id: generateId(),
           role: "user",
           content: [{ type: "text", text: transcript.text }],
-          metadata: { custom: {} },
+          metadata: { modality: "voice", custom: {} },
           createdAt: new Date(),
           status: { type: "complete", reason: "unknown" },
           attachments: [],
-        });
+        };
+        this._voiceMessages.push(message);
+        this._commitVoiceMessage(message);
         this._markVoiceMessagesDirty();
         this._notifySubscribers();
       }
@@ -407,6 +584,7 @@ export abstract class BaseThreadRuntimeCore
             unstable_annotations: [],
             unstable_data: [],
             steps: [],
+            modality: "voice",
             custom: {},
           },
           status,
@@ -426,6 +604,7 @@ export abstract class BaseThreadRuntimeCore
       }
 
       if (transcript.isFinal) {
+        this._commitVoiceMessage(this._currentAssistantMsg);
         this._currentAssistantMsg = null;
       }
 
@@ -434,7 +613,7 @@ export abstract class BaseThreadRuntimeCore
     }
   }
 
-  private _finishVoiceAssistantMessage() {
+  private _finishVoiceAssistantMessage(notify = true) {
     const last = this._voiceMessages.at(-1);
     if (last?.role === "assistant" && last.status.type === "running") {
       const idx = this._voiceMessages.length - 1;
@@ -442,28 +621,51 @@ export abstract class BaseThreadRuntimeCore
         ...(last as ThreadAssistantMessage),
         status: { type: "complete", reason: "stop" },
       };
+      this._commitVoiceMessage(this._voiceMessages[idx]!);
+      this._currentAssistantMsg = null;
       this._markVoiceMessagesDirty();
-      this._notifySubscribers();
+      if (notify) this._notifySubscribers();
     }
   }
 
   public disconnectVoice() {
-    this._finishVoiceAssistantMessage();
+    this._disconnectVoice(true);
+  }
+
+  private _disconnectVoice(fireHook: boolean) {
+    this._finishVoiceAssistantMessage(false);
     this._currentAssistantMsg = null;
-    for (const unsub of this._voiceUnsubs) unsub();
+    // Drain the shared list in place so reentrant setup cannot release the same handles again.
+    const unsubs = this._voiceUnsubs.splice(0);
     this._voiceUnsubs = [];
-    this._voiceSession?.disconnect();
+    const session = this._voiceSession;
     this._voiceSession = undefined;
     this.voice = undefined;
     this._voiceVolume = 0;
-    notifyEventListeners(
-      this._voiceVolumeSubscribers,
-      undefined,
-      "Voice volume",
-    );
+    const stopSpeaking =
+      this.speech && this._isVoiceMessage(this.speech.messageId)
+        ? this._stopSpeaking
+        : undefined;
     this._voiceMessages = [];
     this._markVoiceMessagesDirty();
-    this._notifySubscribers();
+
+    try {
+      notifySubscribers([
+        ...unsubs,
+        ...(stopSpeaking ? [stopSpeaking] : []),
+        ...(session ? [() => session.disconnect()] : []),
+        () =>
+          notifyEventListeners(
+            this._voiceVolumeSubscribers,
+            undefined,
+            "Voice volume",
+          ),
+        () => this._notifySubscribers(),
+      ]);
+    } finally {
+      if (fireHook && session && this._voiceSession === undefined)
+        this._onVoiceDisconnected();
+    }
   }
 
   public muteVoice() {

package/src/runtime/base/default-edit-composer-runtime-core.ts

@@ -14,7 +14,7 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   }
 
   public get canSend() {
-    return !this.isEmpty && !this._isSending;
+    return !this.isEmpty && !this.runtime.voice && !this._isSending;
   }
 
   protected getAttachmentAdapter() {
@@ -52,7 +52,17 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   ) {
     super();
     this.runtime = runtime;
-    this.endEditCallback = endEditCallback;
+    let lastHasVoice = runtime.voice !== undefined;
+    const unsubscribe = runtime.subscribe(() => {
+      const hasVoice = runtime.voice !== undefined;
+      if (hasVoice === lastHasVoice) return;
+      lastHasVoice = hasVoice;
+      this._notifySubscribers();
+    });
+    this.endEditCallback = () => {
+      unsubscribe();
+      endEditCallback();
+    };
     this._parentId = parentId;
     this._sourceId = message.id;
     this.setText(getThreadMessageText(message));

package/src/runtime/base/default-thread-composer-runtime-core.ts

@@ -26,7 +26,12 @@ export class DefaultThreadComposerRuntimeCore
   }
 
   public get canSend() {
-    return !this.isEmpty && !this.runtime.isSendDisabled && !this._isSending;
+    return (
+      !this.isEmpty &&
+      !this.runtime.isSendDisabled &&
+      !this.runtime.voice &&
+      !this._isSending
+    );
   }
 
   private _queueCache:
@@ -99,6 +104,7 @@ export class DefaultThreadComposerRuntimeCore
   public connect() {
     let lastCanCancel = false;
     let lastIsSendDisabled = this.runtime.isSendDisabled;
+    let lastHasVoice = this.runtime.voice !== undefined;
     let lastQueue = this.queue;
     return this.runtime.subscribe(() => {
       let changed = false;
@@ -111,6 +117,11 @@ export class DefaultThreadComposerRuntimeCore
         lastIsSendDisabled = this.runtime.isSendDisabled;
         changed = true;
       }
+      const hasVoice = this.runtime.voice !== undefined;
+      if (lastHasVoice !== hasVoice) {
+        lastHasVoice = hasVoice;
+        changed = true;
+      }
       if (lastQueue !== this.queue) {
         lastQueue = this.queue;
         changed = true;

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -71,6 +71,7 @@ export type RespondToToolApprovalOptions = {
 export type SubmitFeedbackOptions = {
   messageId: string;
   type: "negative" | "positive";
+  comment?: string;
 };
 
 export type ThreadSuggestion = {
@@ -95,9 +96,16 @@ export type VoiceSessionState = {
 
 export type SubmittedFeedback = {
   readonly type: "negative" | "positive";
+  readonly comment?: string;
 };
 
 export type ThreadRuntimeEventPayload = {
+  toolApprovalAnswered: {
+    messageId: string;
+    toolCallId: string;
+    toolName: string;
+    approved: boolean;
+  };
   /**
    * @deprecated State-derivable. Observe `state.isRunning` flipping to `true`
    * via `subscribe` + `getState` instead. Note: this event fires at the