@assistant-ui/core 0.3.19 → 0.3.21

package/src/runtime/base/base-thread-runtime-core.ts

@@ -1,5 +1,6 @@
 import type {
   AppendMessage,
+  TextMessagePart,
   ThreadAssistantMessage,
   ThreadMessage,
 } from "../../types/message";
@@ -11,6 +12,7 @@ import {
   ExportedMessageRepository,
   MessageRepository,
 } from "../utils/message-repository";
+import { captureThreadRuntimeGeneration } from "../utils/thread-runtime-lifecycle";
 import { DefaultThreadComposerRuntimeCore } from "./default-thread-composer-runtime-core";
 import type {
   AddToolResultOptions,
@@ -35,6 +37,7 @@ import type { AttachmentAdapter } from "../../adapters/attachment";
 import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 import { notifyEventListeners } from "../../utils/notify-event-listeners";
+import { MessageNotSentError } from "../../types/error";
 import { gateInteractableComposerMetadata } from "../../model-context/interactable-composer-metadata";
 import {
   BaseSubscribable,
@@ -92,6 +95,25 @@ export abstract class BaseThreadRuntimeCore
     return this.repository.getMessages();
   }
 
+  protected _commitVoiceMessage(
+    _message: ThreadMessage,
+  ): void | Promise<void> {}
+
+  protected _onMessageMetadataChanged(
+    _previousMessage: ThreadAssistantMessage,
+    _message: ThreadAssistantMessage,
+  ): void {}
+
+  protected _dropVoiceMessage(messageId: string, notify: boolean) {
+    const index = this._voiceMessages.findIndex(
+      (voiceMessage) => voiceMessage.id === messageId,
+    );
+    if (index === -1) return;
+    this._voiceMessages.splice(index, 1);
+    this._markVoiceMessagesDirty();
+    if (notify) this._notifySubscribers();
+  }
+
   public get messages(): readonly ThreadMessage[] {
     if (this._voiceMessages.length === 0) {
       return this._getBaseMessages();
@@ -101,7 +123,13 @@ export abstract class BaseThreadRuntimeCore
       this._cachedVoiceGeneration !== this._voiceGeneration ||
       this._cachedMergedBase !== base
     ) {
-      this._cachedMergedMessages = [...base, ...this._voiceMessages];
+      const baseMessageIds = new Set(base.map((message) => message.id));
+      this._cachedMergedMessages = [
+        ...base,
+        ...this._voiceMessages.filter(
+          (message) => !baseMessageIds.has(message.id),
+        ),
+      ];
       this._cachedVoiceGeneration = this._voiceGeneration;
       this._cachedMergedBase = base;
     }
@@ -177,7 +205,17 @@ export abstract class BaseThreadRuntimeCore
     );
   }
 
+  protected _resolveAppendParent(parentId: string | null): string | null {
+    return this._isVoiceMessage(parentId)
+      ? (this._getBaseMessages().at(-1)?.id ?? null)
+      : parentId;
+  }
+
   public beginEdit(messageId: string) {
+    if (this.voice)
+      throw new Error(
+        "Cannot edit a message while a voice session is connected",
+      );
     if (this._isVoiceMessage(messageId)) {
       throw new Error("Voice transcript messages cannot be edited");
     }
@@ -253,21 +291,21 @@ export abstract class BaseThreadRuntimeCore
     });
   }
 
-  public submitFeedback({ messageId, type }: SubmitFeedbackOptions) {
+  public submitFeedback({ messageId, type, comment }: SubmitFeedbackOptions) {
     const adapter = this.adapters?.feedback;
-    if (!adapter) throw new Error("Feedback adapter not configured");
-
     const entry = this.getMessageById(messageId);
     if (!entry) throw new Error(`Message not found: ${messageId}`);
     const { message, parentId } = entry;
-    adapter.submit({ message, type });
+    const trimmed = comment?.trim();
+    const feedback = { type, ...(trimmed ? { comment: trimmed } : undefined) };
+    adapter?.submit({ message, ...feedback });
 
     if (message.role === "assistant") {
-      const updatedMessage: ThreadMessage = {
+      const updatedMessage: ThreadAssistantMessage = {
         ...message,
         metadata: {
           ...message.metadata,
-          submittedFeedback: { type },
+          submittedFeedback: feedback,
         },
       };
       const voiceIdx = this._voiceMessages.findIndex(
@@ -275,10 +313,11 @@ export abstract class BaseThreadRuntimeCore
       );
       if (voiceIdx === -1) {
         this.repository.addOrUpdateMessage(parentId, updatedMessage);
+        this._onMessageMetadataChanged(message, updatedMessage);
       } else {
         this._voiceMessages[voiceIdx] = updatedMessage;
         if (this._currentAssistantMsg === message) {
-          this._currentAssistantMsg = updatedMessage as ThreadAssistantMessage;
+          this._currentAssistantMsg = updatedMessage;
         }
         this._markVoiceMessagesDirty();
       }
@@ -384,12 +423,69 @@ export abstract class BaseThreadRuntimeCore
     return () => this._voiceVolumeSubscribers.delete(callback);
   };
 
+  protected _onVoiceConnected(): void {}
+
+  protected _onVoiceDisconnected(): void {}
+
+  private _toVoiceSessionState(
+    session: RealtimeVoiceAdapter.Session,
+    status: RealtimeVoiceAdapter.Status,
+    mode: RealtimeVoiceAdapter.Mode,
+  ): VoiceSessionState {
+    return {
+      status,
+      isMuted: session.isMuted,
+      mode,
+      canSendText: status.type === "running" && session.sendText !== undefined,
+    };
+  }
+
+  protected _isRunActive(): boolean {
+    const runtime: ThreadRuntimeCore = this;
+    if (runtime.isRunning) return true;
+    const last = this._getBaseMessages().at(-1);
+    return (
+      last?.role === "assistant" &&
+      (last.status.type === "running" || last.status.type === "requires-action")
+    );
+  }
+
+  /**
+   * Waits for a pending history import before a voice message is committed.
+   * The import may begin before or after the voice session connects, so the
+   * loading state must be rechecked when the commit is ready to run. The wait
+   * also ends when the runtime is invalidated, since a superseded runtime may
+   * never learn that loading ended.
+   */
+  protected _getVoiceCommitBarrier(): Promise<void> | undefined {
+    if (!this.isLoading) return undefined;
+    const generation = captureThreadRuntimeGeneration(this);
+    return (async () => {
+      while (this.isLoading && !generation.aborted) {
+        await new Promise<void>((resolve) => {
+          const wake = () => {
+            unsubscribe();
+            generation.removeEventListener("abort", wake);
+            resolve();
+          };
+          const unsubscribe = this.subscribe(wake);
+          generation.addEventListener("abort", wake);
+        });
+      }
+    })();
+  }
+
   public connectVoice() {
     const adapter = this.adapters?.voice;
     if (!adapter) throw new Error("Voice adapter not configured");
+    if (this._isRunActive())
+      throw new Error(
+        "Cannot start a voice session while a run is in progress or paused on a pending tool action",
+      );
+    const replacing = this._voiceSession !== undefined;
 
     try {
-      this.disconnectVoice();
+      this._disconnectVoice(false);
     } catch (error) {
       console.error(
         "[assistant-ui] Voice cleanup threw before reconnect",
@@ -397,7 +493,14 @@ export abstract class BaseThreadRuntimeCore
       );
     }
 
-    const session = adapter.connect({});
+    let session: RealtimeVoiceAdapter.Session;
+    try {
+      session = adapter.connect({});
+    } catch (error) {
+      if (replacing && this._voiceSession === undefined)
+        this._onVoiceDisconnected();
+      throw error;
+    }
     this._voiceSession = session;
     const unsubs: Array<() => void> = [];
     this._voiceUnsubs = unsubs;
@@ -422,27 +525,29 @@ export abstract class BaseThreadRuntimeCore
     try {
       let currentMode: RealtimeVoiceAdapter.Mode = "listening";
 
-      this.voice = {
-        status: session.status,
-        isMuted: session.isMuted,
-        mode: currentMode,
-      };
+      this.voice = this._toVoiceSessionState(
+        session,
+        session.status,
+        currentMode,
+      );
       this._voiceVolume = 0;
       this._notifySubscribers();
       if (finishDetachedSetup()) return;
 
       unsubs.push(
         session.onStatusChange((status) => {
+          if (this._voiceSession !== session) return;
           if (status.type === "ended") {
             this._finishVoiceAssistantMessage();
             this._voiceSession = undefined;
             this.voice = undefined;
+            this._onVoiceDisconnected();
           } else {
-            this.voice = {
+            this.voice = this._toVoiceSessionState(
+              session,
               status,
-              isMuted: session.isMuted,
-              mode: currentMode,
-            };
+              currentMode,
+            );
           }
           this._notifySubscribers();
         }),
@@ -451,6 +556,7 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onModeChange((mode) => {
+          if (this._voiceSession !== session) return;
           currentMode = mode;
           if (this.voice) {
             this.voice = { ...this.voice, mode };
@@ -462,6 +568,7 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onVolumeChange((volume) => {
+          if (this._voiceSession !== session) return;
           this._voiceVolume = volume;
           notifyEventListeners(
             this._voiceVolumeSubscribers,
@@ -474,20 +581,23 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onTranscript((transcript) => {
+          if (this._voiceSession !== session) return;
           this._handleVoiceTranscript(transcript);
         }),
       );
-      finishDetachedSetup();
+      if (!finishDetachedSetup()) this._onVoiceConnected();
     } catch (error) {
       if (this._voiceSession === session && this._voiceUnsubs === unsubs) {
         try {
-          this.disconnectVoice();
+          this._disconnectVoice(false);
         } catch (cleanupError) {
           console.error(
             "[assistant-ui] Voice rollback cleanup threw",
             cleanupError,
           );
         }
+        if (replacing && this._voiceSession === undefined)
+          this._onVoiceDisconnected();
       } else {
         finishDetachedSetup();
       }
@@ -497,6 +607,12 @@ export abstract class BaseThreadRuntimeCore
 
   private _currentAssistantMsg: ThreadAssistantMessage | null = null;
 
+  private _observeVoiceCommit(commit: void | Promise<void>) {
+    void Promise.resolve(commit).catch((error) => {
+      console.error("[assistant-ui] Voice message commit failed", error);
+    });
+  }
+
   private _handleVoiceTranscript(
     transcript: RealtimeVoiceAdapter.TranscriptItem,
   ) {
@@ -507,17 +623,16 @@ export abstract class BaseThreadRuntimeCore
       this._currentAssistantMsg = null;
 
       if (transcript.isFinal) {
-        this._voiceMessages.push({
-          id: generateId(),
-          role: "user",
-          content: [{ type: "text", text: transcript.text }],
-          metadata: { modality: "voice", custom: {} },
-          createdAt: new Date(),
-          status: { type: "complete", reason: "unknown" },
-          attachments: [],
-        });
-        this._markVoiceMessagesDirty();
-        this._notifySubscribers();
+        this._observeVoiceCommit(
+          this._commitVoiceUserMessage({
+            id: generateId(),
+            role: "user",
+            content: [{ type: "text", text: transcript.text }],
+            metadata: { modality: "voice", custom: {} },
+            createdAt: new Date(),
+            attachments: [],
+          }),
+        );
       }
     } else {
       const status: ThreadAssistantMessage["status"] = transcript.isFinal
@@ -554,6 +669,9 @@ export abstract class BaseThreadRuntimeCore
       }
 
       if (transcript.isFinal) {
+        this._observeVoiceCommit(
+          this._commitVoiceMessage(this._currentAssistantMsg),
+        );
         this._currentAssistantMsg = null;
       }
 
@@ -562,7 +680,65 @@ export abstract class BaseThreadRuntimeCore
     }
   }
 
-  private _finishVoiceAssistantMessage() {
+  private _commitVoiceUserMessage(message: ThreadMessage) {
+    this._voiceMessages.push(message);
+    const committed = this._commitVoiceMessage(message);
+    this._markVoiceMessagesDirty();
+    this._notifySubscribers();
+    return committed;
+  }
+
+  protected async _appendToVoiceSession(message: AppendMessage) {
+    const session = this._voiceSession;
+    if (!this.voice?.canSendText || !session?.sendText)
+      throw new Error(
+        "Cannot send a text message while a voice session is connected",
+      );
+    const content = message.content.filter(
+      (part): part is TextMessagePart => part.type === "text",
+    );
+    if (
+      message.role !== "user" ||
+      message.sourceId != null ||
+      message.parentId !==
+        this._resolveAppendParent(this.messages.at(-1)?.id ?? null) ||
+      message.attachments?.length ||
+      content.length !== message.content.length ||
+      !content.some((part) => part.text.trim())
+    )
+      throw new Error(
+        "Only a plain text user message can be sent while a voice session is connected",
+      );
+
+    const enriched = this.enrichAppendMetadata(message);
+    this.ensureInitialized();
+    const generation = captureThreadRuntimeGeneration(this);
+    try {
+      await session.sendText(getThreadMessageText(message));
+    } catch (error) {
+      if (generation.aborted) return;
+      const notSent = new MessageNotSentError();
+      notSent.cause = error;
+      throw notSent;
+    }
+    if (generation.aborted) return;
+    if (this._voiceSession !== session)
+      throw new MessageNotSentError(
+        "The voice session ended before the typed message was recorded",
+      );
+    this._finishVoiceAssistantMessage(false);
+    this._currentAssistantMsg = null;
+    await this._commitVoiceUserMessage({
+      id: generateId(),
+      role: "user",
+      content,
+      metadata: { custom: { ...enriched.metadata?.custom } },
+      createdAt: message.createdAt,
+      attachments: [],
+    });
+  }
+
+  private _finishVoiceAssistantMessage(notify = true) {
     const last = this._voiceMessages.at(-1);
     if (last?.role === "assistant" && last.status.type === "running") {
       const idx = this._voiceMessages.length - 1;
@@ -570,12 +746,21 @@ export abstract class BaseThreadRuntimeCore
         ...(last as ThreadAssistantMessage),
         status: { type: "complete", reason: "stop" },
       };
+      this._observeVoiceCommit(
+        this._commitVoiceMessage(this._voiceMessages[idx]!),
+      );
+      this._currentAssistantMsg = null;
       this._markVoiceMessagesDirty();
-      this._notifySubscribers();
+      if (notify) this._notifySubscribers();
     }
   }
 
   public disconnectVoice() {
+    this._disconnectVoice(true);
+  }
+
+  private _disconnectVoice(fireHook: boolean) {
+    this._finishVoiceAssistantMessage(false);
     this._currentAssistantMsg = null;
     // Drain the shared list in place so reentrant setup cannot release the same handles again.
     const unsubs = this._voiceUnsubs.splice(0);
@@ -591,18 +776,23 @@ export abstract class BaseThreadRuntimeCore
     this._voiceMessages = [];
     this._markVoiceMessagesDirty();
 
-    notifySubscribers([
-      ...unsubs,
-      ...(stopSpeaking ? [stopSpeaking] : []),
-      ...(session ? [() => session.disconnect()] : []),
-      () =>
-        notifyEventListeners(
-          this._voiceVolumeSubscribers,
-          undefined,
-          "Voice volume",
-        ),
-      () => this._notifySubscribers(),
-    ]);
+    try {
+      notifySubscribers([
+        ...unsubs,
+        ...(stopSpeaking ? [stopSpeaking] : []),
+        ...(session ? [() => session.disconnect()] : []),
+        () =>
+          notifyEventListeners(
+            this._voiceVolumeSubscribers,
+            undefined,
+            "Voice volume",
+          ),
+        () => this._notifySubscribers(),
+      ]);
+    } finally {
+      if (fireHook && session && this._voiceSession === undefined)
+        this._onVoiceDisconnected();
+    }
   }
 
   public muteVoice() {

package/src/runtime/base/default-edit-composer-runtime-core.ts

@@ -13,8 +13,14 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
     return true;
   }
 
+  // An edit composer is the message it is editing, so its draft stays in place
+  // until the edit lands rather than moving into a submission row.
+  protected override get detachesDraftOnSend() {
+    return false;
+  }
+
   public get canSend() {
-    return !this.isEmpty && !this._isSending;
+    return !this.isEmpty && !this.runtime.voice && !this.isSubmitting;
   }
 
   protected getAttachmentAdapter() {
@@ -37,6 +43,7 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
       | undefined;
   };
   private endEditCallback: () => void;
+  private _ended = false;
 
   constructor(
     runtime: ThreadRuntimeCore & {
@@ -52,7 +59,17 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   ) {
     super();
     this.runtime = runtime;
-    this.endEditCallback = endEditCallback;
+    let lastHasVoice = runtime.voice !== undefined;
+    const unsubscribe = runtime.subscribe(() => {
+      const hasVoice = runtime.voice !== undefined;
+      if (hasVoice === lastHasVoice) return;
+      lastHasVoice = hasVoice;
+      this._notifySubscribers();
+    });
+    this.endEditCallback = () => {
+      unsubscribe();
+      endEditCallback();
+    };
     this._parentId = parentId;
     this._sourceId = message.id;
     this.setText(getThreadMessageText(message));
@@ -109,6 +126,11 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   }
 
   public handleCancel() {
+    if (this._ended) return;
+    this._ended = true;
+    void this.reset().catch((error: unknown) => {
+      console.error("[assistant-ui] Failed to clear cancelled edit", error);
+    });
     this.endEditCallback();
     this._notifySubscribers();
   }

package/src/runtime/base/default-thread-composer-runtime-core.ts

@@ -1,4 +1,4 @@
-import type { AppendMessage } from "../../types/message";
+import type { AppendMessage, MessageRole } from "../../types/message";
 import type { AttachmentAdapter } from "../../adapters/attachment";
 import type { DictationAdapter } from "../../adapters/speech";
 import type {
@@ -22,11 +22,34 @@ export class DefaultThreadComposerRuntimeCore
   implements ThreadComposerRuntimeCore
 {
   public get canCancel() {
-    return isCancelable(this.runtime);
+    return this.isSubmitting || isCancelable(this.runtime);
   }
 
   public get canSend() {
-    return !this.isEmpty && !this.runtime.isSendDisabled && !this._isSending;
+    if (this.isEmpty || this.runtime.isSendDisabled || this.isSubmitting)
+      return false;
+    const voice = this.runtime.voice;
+    if (!voice) return true;
+    return (
+      voice.canSendText && this.role === "user" && this.attachments.length === 0
+    );
+  }
+
+  public override cancel() {
+    if (!this.isSubmitting) {
+      super.cancel();
+      return;
+    }
+    // Stopping takes a send still being prepared back into the draft and
+    // still stops a run that is going, so the one control never loses either.
+    this.cancelSubmission();
+    if (isCancelable(this.runtime)) super.cancel();
+  }
+
+  protected override threadMessageIds(role: MessageRole) {
+    return this.runtime.messages
+      .filter((message) => message.role === role)
+      .map((message) => message.id);
   }
 
   private _queueCache:
@@ -99,8 +122,10 @@ export class DefaultThreadComposerRuntimeCore
   public connect() {
     let lastCanCancel = false;
     let lastIsSendDisabled = this.runtime.isSendDisabled;
+    let lastVoiceInput = this.runtime.voice?.canSendText;
     let lastQueue = this.queue;
     return this.runtime.subscribe(() => {
+      this.settleInTransit();
       let changed = false;
       const nextCanCancel = this.canCancel;
       if (lastCanCancel !== nextCanCancel) {
@@ -111,6 +136,11 @@ export class DefaultThreadComposerRuntimeCore
         lastIsSendDisabled = this.runtime.isSendDisabled;
         changed = true;
       }
+      const nextVoiceInput = this.runtime.voice?.canSendText;
+      if (lastVoiceInput !== nextVoiceInput) {
+        lastVoiceInput = nextVoiceInput;
+        changed = true;
+      }
       if (lastQueue !== this.queue) {
         lastQueue = this.queue;
         changed = true;

package/src/runtime/interfaces/composer-runtime-core.ts

@@ -50,6 +50,20 @@ export type DictationState = {
   readonly inputDisabled?: boolean;
 };
 
+/**
+ * A message the user sent that the thread does not show as one of its own
+ * yet, because its attachments are still being prepared or the runtime has
+ * not shown it since taking it. A send that cannot be delivered takes its
+ * content back into the draft, so a submission is always in flight.
+ */
+export type ComposerSubmission = {
+  readonly id: string;
+  readonly role: MessageRole;
+  readonly text: string;
+  readonly quote: QuoteInfo | undefined;
+  readonly attachments: readonly Attachment[];
+};
+
 export type SendOptions = {
   startRun?: boolean;
   /** Process this message next; only meaningful with the `queue` capability. */
@@ -87,6 +101,11 @@ export type ComposerRuntimeCore = Readonly<{
   send: (options?: SendOptions) => void;
   cancel: () => void;
 
+  /** The message this composer sent while its attachments are prepared. */
+  submission?: ComposerSubmission | undefined;
+  /** Messages this composer handed to the runtime that the thread does not show yet. */
+  inTransit?: readonly ComposerSubmission[] | undefined;
+
   queue: readonly QueueItemState[];
   moveQueueItem: (queueItemId: string, placement: QueuePlacement) => void;
   removeQueueItem: (queueItemId: string) => void;

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -2,7 +2,11 @@ import type { ToolModelContentPart } from "assistant-stream";
 import type { ReadonlyJSONValue } from "assistant-stream/utils";
 import type { ModelContext } from "../../model-context/types";
 import type { Unsubscribe } from "../../types/unsubscribe";
-import type { AppendMessage, ThreadMessage } from "../../types/message";
+import type {
+  AppendMessage,
+  ThreadMessage,
+  Unstable_ToolInteraction,
+} from "../../types/message";
 import type { RunConfig } from "../../types/message";
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
 import type { RealtimeVoiceAdapter } from "../../adapters/voice";
@@ -58,6 +62,12 @@ export type ResumeToolCallOptions = {
   payload: unknown;
 };
 
+export type Unstable_RecordToolInteractionOptions = {
+  messageId: string;
+  toolCallId: string;
+  interaction: Unstable_ToolInteraction;
+};
+
 export type RespondToToolApprovalOptions = {
   approvalId: string;
   approved: boolean;
@@ -71,6 +81,7 @@ export type RespondToToolApprovalOptions = {
 export type SubmitFeedbackOptions = {
   messageId: string;
   type: "negative" | "positive";
+  comment?: string;
 };
 
 export type ThreadSuggestion = {
@@ -91,10 +102,15 @@ export type VoiceSessionState = {
   readonly status: RealtimeVoiceAdapter.Status;
   readonly isMuted: boolean;
   readonly mode: RealtimeVoiceAdapter.Mode;
+  /**
+   * Whether the running session takes typed text. While true, `append` routes a plain text user message into the session and the thread composer can send.
+   */
+  readonly canSendText: boolean;
 };
 
 export type SubmittedFeedback = {
   readonly type: "negative" | "positive";
+  readonly comment?: string;
 };
 
 export type ThreadRuntimeEventPayload = {
@@ -188,6 +204,13 @@ export type ThreadRuntimeCore = Readonly<{
   respondToToolApproval: (
     options: RespondToToolApprovalOptions,
   ) => Promise<void>;
+  /**
+   * Appends a validated interaction to a tool call part and persists it where
+   * the runtime persists messages. Rejects when the runtime cannot record it.
+   */
+  unstable_recordToolInteraction?: (
+    options: Unstable_RecordToolInteractionOptions,
+  ) => Promise<void>;
 
   speak: (messageId: string) => void;
   stopSpeaking: () => void;