@assistant-ui/core 0.3.20 → 0.3.21

package/src/runtime/base/base-thread-runtime-core.ts

@@ -1,5 +1,6 @@
 import type {
   AppendMessage,
+  TextMessagePart,
   ThreadAssistantMessage,
   ThreadMessage,
 } from "../../types/message";
@@ -11,6 +12,7 @@ import {
   ExportedMessageRepository,
   MessageRepository,
 } from "../utils/message-repository";
+import { captureThreadRuntimeGeneration } from "../utils/thread-runtime-lifecycle";
 import { DefaultThreadComposerRuntimeCore } from "./default-thread-composer-runtime-core";
 import type {
   AddToolResultOptions,
@@ -35,6 +37,7 @@ import type { AttachmentAdapter } from "../../adapters/attachment";
 import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 import { notifyEventListeners } from "../../utils/notify-event-listeners";
+import { MessageNotSentError } from "../../types/error";
 import { gateInteractableComposerMetadata } from "../../model-context/interactable-composer-metadata";
 import {
   BaseSubscribable,
@@ -92,7 +95,24 @@ export abstract class BaseThreadRuntimeCore
     return this.repository.getMessages();
   }
 
-  protected _commitVoiceMessage(_message: ThreadMessage): void {}
+  protected _commitVoiceMessage(
+    _message: ThreadMessage,
+  ): void | Promise<void> {}
+
+  protected _onMessageMetadataChanged(
+    _previousMessage: ThreadAssistantMessage,
+    _message: ThreadAssistantMessage,
+  ): void {}
+
+  protected _dropVoiceMessage(messageId: string, notify: boolean) {
+    const index = this._voiceMessages.findIndex(
+      (voiceMessage) => voiceMessage.id === messageId,
+    );
+    if (index === -1) return;
+    this._voiceMessages.splice(index, 1);
+    this._markVoiceMessagesDirty();
+    if (notify) this._notifySubscribers();
+  }
 
   public get messages(): readonly ThreadMessage[] {
     if (this._voiceMessages.length === 0) {
@@ -281,7 +301,7 @@ export abstract class BaseThreadRuntimeCore
     adapter?.submit({ message, ...feedback });
 
     if (message.role === "assistant") {
-      const updatedMessage: ThreadMessage = {
+      const updatedMessage: ThreadAssistantMessage = {
         ...message,
         metadata: {
           ...message.metadata,
@@ -293,10 +313,11 @@ export abstract class BaseThreadRuntimeCore
       );
       if (voiceIdx === -1) {
         this.repository.addOrUpdateMessage(parentId, updatedMessage);
+        this._onMessageMetadataChanged(message, updatedMessage);
       } else {
         this._voiceMessages[voiceIdx] = updatedMessage;
         if (this._currentAssistantMsg === message) {
-          this._currentAssistantMsg = updatedMessage as ThreadAssistantMessage;
+          this._currentAssistantMsg = updatedMessage;
         }
         this._markVoiceMessagesDirty();
       }
@@ -406,6 +427,19 @@ export abstract class BaseThreadRuntimeCore
 
   protected _onVoiceDisconnected(): void {}
 
+  private _toVoiceSessionState(
+    session: RealtimeVoiceAdapter.Session,
+    status: RealtimeVoiceAdapter.Status,
+    mode: RealtimeVoiceAdapter.Mode,
+  ): VoiceSessionState {
+    return {
+      status,
+      isMuted: session.isMuted,
+      mode,
+      canSendText: status.type === "running" && session.sendText !== undefined,
+    };
+  }
+
   protected _isRunActive(): boolean {
     const runtime: ThreadRuntimeCore = this;
     if (runtime.isRunning) return true;
@@ -416,6 +450,31 @@ export abstract class BaseThreadRuntimeCore
     );
   }
 
+  /**
+   * Waits for a pending history import before a voice message is committed.
+   * The import may begin before or after the voice session connects, so the
+   * loading state must be rechecked when the commit is ready to run. The wait
+   * also ends when the runtime is invalidated, since a superseded runtime may
+   * never learn that loading ended.
+   */
+  protected _getVoiceCommitBarrier(): Promise<void> | undefined {
+    if (!this.isLoading) return undefined;
+    const generation = captureThreadRuntimeGeneration(this);
+    return (async () => {
+      while (this.isLoading && !generation.aborted) {
+        await new Promise<void>((resolve) => {
+          const wake = () => {
+            unsubscribe();
+            generation.removeEventListener("abort", wake);
+            resolve();
+          };
+          const unsubscribe = this.subscribe(wake);
+          generation.addEventListener("abort", wake);
+        });
+      }
+    })();
+  }
+
   public connectVoice() {
     const adapter = this.adapters?.voice;
     if (!adapter) throw new Error("Voice adapter not configured");
@@ -466,11 +525,11 @@ export abstract class BaseThreadRuntimeCore
     try {
       let currentMode: RealtimeVoiceAdapter.Mode = "listening";
 
-      this.voice = {
-        status: session.status,
-        isMuted: session.isMuted,
-        mode: currentMode,
-      };
+      this.voice = this._toVoiceSessionState(
+        session,
+        session.status,
+        currentMode,
+      );
       this._voiceVolume = 0;
       this._notifySubscribers();
       if (finishDetachedSetup()) return;
@@ -484,11 +543,11 @@ export abstract class BaseThreadRuntimeCore
             this.voice = undefined;
             this._onVoiceDisconnected();
           } else {
-            this.voice = {
+            this.voice = this._toVoiceSessionState(
+              session,
               status,
-              isMuted: session.isMuted,
-              mode: currentMode,
-            };
+              currentMode,
+            );
           }
           this._notifySubscribers();
         }),
@@ -497,6 +556,7 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onModeChange((mode) => {
+          if (this._voiceSession !== session) return;
           currentMode = mode;
           if (this.voice) {
             this.voice = { ...this.voice, mode };
@@ -508,6 +568,7 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onVolumeChange((volume) => {
+          if (this._voiceSession !== session) return;
           this._voiceVolume = volume;
           notifyEventListeners(
             this._voiceVolumeSubscribers,
@@ -520,6 +581,7 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onTranscript((transcript) => {
+          if (this._voiceSession !== session) return;
           this._handleVoiceTranscript(transcript);
         }),
       );
@@ -545,6 +607,12 @@ export abstract class BaseThreadRuntimeCore
 
   private _currentAssistantMsg: ThreadAssistantMessage | null = null;
 
+  private _observeVoiceCommit(commit: void | Promise<void>) {
+    void Promise.resolve(commit).catch((error) => {
+      console.error("[assistant-ui] Voice message commit failed", error);
+    });
+  }
+
   private _handleVoiceTranscript(
     transcript: RealtimeVoiceAdapter.TranscriptItem,
   ) {
@@ -555,19 +623,16 @@ export abstract class BaseThreadRuntimeCore
       this._currentAssistantMsg = null;
 
       if (transcript.isFinal) {
-        const message: ThreadMessage = {
-          id: generateId(),
-          role: "user",
-          content: [{ type: "text", text: transcript.text }],
-          metadata: { modality: "voice", custom: {} },
-          createdAt: new Date(),
-          status: { type: "complete", reason: "unknown" },
-          attachments: [],
-        };
-        this._voiceMessages.push(message);
-        this._commitVoiceMessage(message);
-        this._markVoiceMessagesDirty();
-        this._notifySubscribers();
+        this._observeVoiceCommit(
+          this._commitVoiceUserMessage({
+            id: generateId(),
+            role: "user",
+            content: [{ type: "text", text: transcript.text }],
+            metadata: { modality: "voice", custom: {} },
+            createdAt: new Date(),
+            attachments: [],
+          }),
+        );
       }
     } else {
       const status: ThreadAssistantMessage["status"] = transcript.isFinal
@@ -604,7 +669,9 @@ export abstract class BaseThreadRuntimeCore
       }
 
       if (transcript.isFinal) {
-        this._commitVoiceMessage(this._currentAssistantMsg);
+        this._observeVoiceCommit(
+          this._commitVoiceMessage(this._currentAssistantMsg),
+        );
         this._currentAssistantMsg = null;
       }
 
@@ -613,6 +680,64 @@ export abstract class BaseThreadRuntimeCore
     }
   }
 
+  private _commitVoiceUserMessage(message: ThreadMessage) {
+    this._voiceMessages.push(message);
+    const committed = this._commitVoiceMessage(message);
+    this._markVoiceMessagesDirty();
+    this._notifySubscribers();
+    return committed;
+  }
+
+  protected async _appendToVoiceSession(message: AppendMessage) {
+    const session = this._voiceSession;
+    if (!this.voice?.canSendText || !session?.sendText)
+      throw new Error(
+        "Cannot send a text message while a voice session is connected",
+      );
+    const content = message.content.filter(
+      (part): part is TextMessagePart => part.type === "text",
+    );
+    if (
+      message.role !== "user" ||
+      message.sourceId != null ||
+      message.parentId !==
+        this._resolveAppendParent(this.messages.at(-1)?.id ?? null) ||
+      message.attachments?.length ||
+      content.length !== message.content.length ||
+      !content.some((part) => part.text.trim())
+    )
+      throw new Error(
+        "Only a plain text user message can be sent while a voice session is connected",
+      );
+
+    const enriched = this.enrichAppendMetadata(message);
+    this.ensureInitialized();
+    const generation = captureThreadRuntimeGeneration(this);
+    try {
+      await session.sendText(getThreadMessageText(message));
+    } catch (error) {
+      if (generation.aborted) return;
+      const notSent = new MessageNotSentError();
+      notSent.cause = error;
+      throw notSent;
+    }
+    if (generation.aborted) return;
+    if (this._voiceSession !== session)
+      throw new MessageNotSentError(
+        "The voice session ended before the typed message was recorded",
+      );
+    this._finishVoiceAssistantMessage(false);
+    this._currentAssistantMsg = null;
+    await this._commitVoiceUserMessage({
+      id: generateId(),
+      role: "user",
+      content,
+      metadata: { custom: { ...enriched.metadata?.custom } },
+      createdAt: message.createdAt,
+      attachments: [],
+    });
+  }
+
   private _finishVoiceAssistantMessage(notify = true) {
     const last = this._voiceMessages.at(-1);
     if (last?.role === "assistant" && last.status.type === "running") {
@@ -621,7 +746,9 @@ export abstract class BaseThreadRuntimeCore
         ...(last as ThreadAssistantMessage),
         status: { type: "complete", reason: "stop" },
       };
-      this._commitVoiceMessage(this._voiceMessages[idx]!);
+      this._observeVoiceCommit(
+        this._commitVoiceMessage(this._voiceMessages[idx]!),
+      );
       this._currentAssistantMsg = null;
       this._markVoiceMessagesDirty();
       if (notify) this._notifySubscribers();

package/src/runtime/base/default-edit-composer-runtime-core.ts

@@ -13,8 +13,14 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
     return true;
   }
 
+  // An edit composer is the message it is editing, so its draft stays in place
+  // until the edit lands rather than moving into a submission row.
+  protected override get detachesDraftOnSend() {
+    return false;
+  }
+
   public get canSend() {
-    return !this.isEmpty && !this.runtime.voice && !this._isSending;
+    return !this.isEmpty && !this.runtime.voice && !this.isSubmitting;
   }
 
   protected getAttachmentAdapter() {
@@ -37,6 +43,7 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
       | undefined;
   };
   private endEditCallback: () => void;
+  private _ended = false;
 
   constructor(
     runtime: ThreadRuntimeCore & {
@@ -119,6 +126,11 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   }
 
   public handleCancel() {
+    if (this._ended) return;
+    this._ended = true;
+    void this.reset().catch((error: unknown) => {
+      console.error("[assistant-ui] Failed to clear cancelled edit", error);
+    });
     this.endEditCallback();
     this._notifySubscribers();
   }

package/src/runtime/base/default-thread-composer-runtime-core.ts

@@ -1,4 +1,4 @@
-import type { AppendMessage } from "../../types/message";
+import type { AppendMessage, MessageRole } from "../../types/message";
 import type { AttachmentAdapter } from "../../adapters/attachment";
 import type { DictationAdapter } from "../../adapters/speech";
 import type {
@@ -22,18 +22,36 @@ export class DefaultThreadComposerRuntimeCore
   implements ThreadComposerRuntimeCore
 {
   public get canCancel() {
-    return isCancelable(this.runtime);
+    return this.isSubmitting || isCancelable(this.runtime);
   }
 
   public get canSend() {
+    if (this.isEmpty || this.runtime.isSendDisabled || this.isSubmitting)
+      return false;
+    const voice = this.runtime.voice;
+    if (!voice) return true;
     return (
-      !this.isEmpty &&
-      !this.runtime.isSendDisabled &&
-      !this.runtime.voice &&
-      !this._isSending
+      voice.canSendText && this.role === "user" && this.attachments.length === 0
     );
   }
 
+  public override cancel() {
+    if (!this.isSubmitting) {
+      super.cancel();
+      return;
+    }
+    // Stopping takes a send still being prepared back into the draft and
+    // still stops a run that is going, so the one control never loses either.
+    this.cancelSubmission();
+    if (isCancelable(this.runtime)) super.cancel();
+  }
+
+  protected override threadMessageIds(role: MessageRole) {
+    return this.runtime.messages
+      .filter((message) => message.role === role)
+      .map((message) => message.id);
+  }
+
   private _queueCache:
     | {
         steer: readonly QueueItemState[];
@@ -104,9 +122,10 @@ export class DefaultThreadComposerRuntimeCore
   public connect() {
     let lastCanCancel = false;
     let lastIsSendDisabled = this.runtime.isSendDisabled;
-    let lastHasVoice = this.runtime.voice !== undefined;
+    let lastVoiceInput = this.runtime.voice?.canSendText;
     let lastQueue = this.queue;
     return this.runtime.subscribe(() => {
+      this.settleInTransit();
       let changed = false;
       const nextCanCancel = this.canCancel;
       if (lastCanCancel !== nextCanCancel) {
@@ -117,9 +136,9 @@ export class DefaultThreadComposerRuntimeCore
         lastIsSendDisabled = this.runtime.isSendDisabled;
         changed = true;
       }
-      const hasVoice = this.runtime.voice !== undefined;
-      if (lastHasVoice !== hasVoice) {
-        lastHasVoice = hasVoice;
+      const nextVoiceInput = this.runtime.voice?.canSendText;
+      if (lastVoiceInput !== nextVoiceInput) {
+        lastVoiceInput = nextVoiceInput;
         changed = true;
       }
       if (lastQueue !== this.queue) {

package/src/runtime/interfaces/composer-runtime-core.ts

@@ -50,6 +50,20 @@ export type DictationState = {
   readonly inputDisabled?: boolean;
 };
 
+/**
+ * A message the user sent that the thread does not show as one of its own
+ * yet, because its attachments are still being prepared or the runtime has
+ * not shown it since taking it. A send that cannot be delivered takes its
+ * content back into the draft, so a submission is always in flight.
+ */
+export type ComposerSubmission = {
+  readonly id: string;
+  readonly role: MessageRole;
+  readonly text: string;
+  readonly quote: QuoteInfo | undefined;
+  readonly attachments: readonly Attachment[];
+};
+
 export type SendOptions = {
   startRun?: boolean;
   /** Process this message next; only meaningful with the `queue` capability. */
@@ -87,6 +101,11 @@ export type ComposerRuntimeCore = Readonly<{
   send: (options?: SendOptions) => void;
   cancel: () => void;
 
+  /** The message this composer sent while its attachments are prepared. */
+  submission?: ComposerSubmission | undefined;
+  /** Messages this composer handed to the runtime that the thread does not show yet. */
+  inTransit?: readonly ComposerSubmission[] | undefined;
+
   queue: readonly QueueItemState[];
   moveQueueItem: (queueItemId: string, placement: QueuePlacement) => void;
   removeQueueItem: (queueItemId: string) => void;

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -2,7 +2,11 @@ import type { ToolModelContentPart } from "assistant-stream";
 import type { ReadonlyJSONValue } from "assistant-stream/utils";
 import type { ModelContext } from "../../model-context/types";
 import type { Unsubscribe } from "../../types/unsubscribe";
-import type { AppendMessage, ThreadMessage } from "../../types/message";
+import type {
+  AppendMessage,
+  ThreadMessage,
+  Unstable_ToolInteraction,
+} from "../../types/message";
 import type { RunConfig } from "../../types/message";
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
 import type { RealtimeVoiceAdapter } from "../../adapters/voice";
@@ -58,6 +62,12 @@ export type ResumeToolCallOptions = {
   payload: unknown;
 };
 
+export type Unstable_RecordToolInteractionOptions = {
+  messageId: string;
+  toolCallId: string;
+  interaction: Unstable_ToolInteraction;
+};
+
 export type RespondToToolApprovalOptions = {
   approvalId: string;
   approved: boolean;
@@ -92,6 +102,10 @@ export type VoiceSessionState = {
   readonly status: RealtimeVoiceAdapter.Status;
   readonly isMuted: boolean;
   readonly mode: RealtimeVoiceAdapter.Mode;
+  /**
+   * Whether the running session takes typed text. While true, `append` routes a plain text user message into the session and the thread composer can send.
+   */
+  readonly canSendText: boolean;
 };
 
 export type SubmittedFeedback = {
@@ -190,6 +204,13 @@ export type ThreadRuntimeCore = Readonly<{
   respondToToolApproval: (
     options: RespondToToolApprovalOptions,
   ) => Promise<void>;
+  /**
+   * Appends a validated interaction to a tool call part and persists it where
+   * the runtime persists messages. Rejects when the runtime cannot record it.
+   */
+  unstable_recordToolInteraction?: (
+    options: Unstable_RecordToolInteractionOptions,
+  ) => Promise<void>;
 
   speak: (messageId: string) => void;
   stopSpeaking: () => void;

package/src/runtime/utils/attachment-add-operations.ts

@@ -1,3 +1,5 @@
+import type { Attachment } from "../../types/attachment";
+
 export type AttachmentAddOperation = {
   cancelled: boolean;
   attachmentIds: Set<string>;
@@ -5,6 +7,10 @@ export type AttachmentAddOperation = {
 
 export class AttachmentAddOperations {
   private readonly operations = new Set<AttachmentAddOperation>();
+  private readonly uploading = new Map<
+    string,
+    { operation: AttachmentAddOperation; waiters: Set<() => void> }
+  >();
 
   start() {
     const operation: AttachmentAddOperation = {
@@ -15,14 +21,32 @@ export class AttachmentAddOperations {
     return operation;
   }
 
-  accept(operation: AttachmentAddOperation, attachmentId: string) {
+  accept(
+    operation: AttachmentAddOperation,
+    attachment: Pick<Attachment, "id" | "status">,
+  ) {
     if (operation.cancelled) return false;
-    operation.attachmentIds.add(attachmentId);
+    operation.attachmentIds.add(attachment.id);
+    // The composer shows one attachment per id, so the add that updated it
+    // last owns its upload; an add that ended earlier no longer speaks for it.
+    const entry = this.uploading.get(attachment.id);
+    if (
+      entry?.operation !== operation &&
+      (entry || attachment.status.type === "running")
+    )
+      this.uploading.set(attachment.id, {
+        operation,
+        waiters: entry?.waiters ?? new Set(),
+      });
+    if (attachment.status.type !== "running")
+      this.settle(attachment.id, operation);
     return true;
   }
 
   finish(operation: AttachmentAddOperation) {
     this.operations.delete(operation);
+    for (const attachmentId of operation.attachmentIds)
+      this.settle(attachmentId, operation);
   }
 
   isCancelled(operation: AttachmentAddOperation) {
@@ -35,6 +59,7 @@ export class AttachmentAddOperations {
       operation.cancelled = true;
       this.operations.delete(operation);
     }
+    this.settle(attachmentId);
   }
 
   cancelAll() {
@@ -42,6 +67,24 @@ export class AttachmentAddOperations {
       operation.cancelled = true;
     }
     this.operations.clear();
+    for (const attachmentId of [...this.uploading.keys()])
+      this.settle(attachmentId);
+  }
+
+  whenSendable(attachmentId: string): Promise<void> | undefined {
+    const entry = this.uploading.get(attachmentId);
+    if (!entry) return undefined;
+    return new Promise((resolve) => entry.waiters.add(resolve));
+  }
+
+  // An attachment id outlives the add that produced it, so an add that ends
+  // after a newer one took the id over must not release its upload.
+  private settle(attachmentId: string, operation?: AttachmentAddOperation) {
+    const entry = this.uploading.get(attachmentId);
+    if (!entry) return;
+    if (operation && entry.operation !== operation) return;
+    this.uploading.delete(attachmentId);
+    for (const resolve of entry.waiters) resolve();
   }
 }
 

package/src/runtime/utils/attachment-send-operations.ts

@@ -0,0 +1,54 @@
+import type { AttachmentAdapter } from "../../adapters/attachment";
+import {
+  isAttachmentComplete,
+  type Attachment,
+  type CompleteAttachment,
+} from "../../types/attachment";
+
+export class AttachmentSendOperations {
+  // Draft attachments stay pending so discarding an unsent upload still calls remove.
+  private readonly entries = new WeakMap<
+    Attachment,
+    { result?: CompleteAttachment }
+  >();
+  // Removal marks are per-object and never bulk-cleared: a removed attachment
+  // either leaves the draft or is replaced via transfer with a fresh unmarked
+  // object, so a mark cannot leak into a later send's batch.
+  private readonly removed = new WeakSet<Attachment>();
+
+  markRemoved(attachment: Attachment) {
+    this.removed.add(attachment);
+  }
+
+  unmarkRemoved(attachment: Attachment) {
+    this.removed.delete(attachment);
+  }
+
+  isRemoved(attachment: Attachment) {
+    return this.removed.has(attachment);
+  }
+
+  async send(
+    attachment: Attachment,
+    adapter: AttachmentAdapter | undefined,
+    signal?: AbortSignal,
+  ): Promise<CompleteAttachment> {
+    if (isAttachmentComplete(attachment)) return attachment;
+    const entry = this.entries.get(attachment) ?? {};
+    if (entry.result) return entry.result;
+    if (!adapter) throw new Error("Attachments are not supported");
+    this.entries.set(attachment, entry);
+    const result = await adapter.send(
+      attachment,
+      signal ? { signal } : undefined,
+    );
+    entry.result = result;
+    return result;
+  }
+
+  transfer(original: Attachment, replacement: Attachment) {
+    const entry = this.entries.get(original);
+    if (entry) this.entries.set(replacement, entry);
+    return replacement;
+  }
+}