@assistant-ui/core 0.3.19 → 0.3.20

package/src/runtime/base/base-thread-runtime-core.ts

@@ -92,6 +92,8 @@ export abstract class BaseThreadRuntimeCore
     return this.repository.getMessages();
   }
 
+  protected _commitVoiceMessage(_message: ThreadMessage): void {}
+
   public get messages(): readonly ThreadMessage[] {
     if (this._voiceMessages.length === 0) {
       return this._getBaseMessages();
@@ -101,7 +103,13 @@ export abstract class BaseThreadRuntimeCore
       this._cachedVoiceGeneration !== this._voiceGeneration ||
       this._cachedMergedBase !== base
     ) {
-      this._cachedMergedMessages = [...base, ...this._voiceMessages];
+      const baseMessageIds = new Set(base.map((message) => message.id));
+      this._cachedMergedMessages = [
+        ...base,
+        ...this._voiceMessages.filter(
+          (message) => !baseMessageIds.has(message.id),
+        ),
+      ];
       this._cachedVoiceGeneration = this._voiceGeneration;
       this._cachedMergedBase = base;
     }
@@ -177,7 +185,17 @@ export abstract class BaseThreadRuntimeCore
     );
   }
 
+  protected _resolveAppendParent(parentId: string | null): string | null {
+    return this._isVoiceMessage(parentId)
+      ? (this._getBaseMessages().at(-1)?.id ?? null)
+      : parentId;
+  }
+
   public beginEdit(messageId: string) {
+    if (this.voice)
+      throw new Error(
+        "Cannot edit a message while a voice session is connected",
+      );
     if (this._isVoiceMessage(messageId)) {
       throw new Error("Voice transcript messages cannot be edited");
     }
@@ -253,21 +271,21 @@ export abstract class BaseThreadRuntimeCore
     });
   }
 
-  public submitFeedback({ messageId, type }: SubmitFeedbackOptions) {
+  public submitFeedback({ messageId, type, comment }: SubmitFeedbackOptions) {
     const adapter = this.adapters?.feedback;
-    if (!adapter) throw new Error("Feedback adapter not configured");
-
     const entry = this.getMessageById(messageId);
     if (!entry) throw new Error(`Message not found: ${messageId}`);
     const { message, parentId } = entry;
-    adapter.submit({ message, type });
+    const trimmed = comment?.trim();
+    const feedback = { type, ...(trimmed ? { comment: trimmed } : undefined) };
+    adapter?.submit({ message, ...feedback });
 
     if (message.role === "assistant") {
       const updatedMessage: ThreadMessage = {
         ...message,
         metadata: {
           ...message.metadata,
-          submittedFeedback: { type },
+          submittedFeedback: feedback,
         },
       };
       const voiceIdx = this._voiceMessages.findIndex(
@@ -384,12 +402,31 @@ export abstract class BaseThreadRuntimeCore
     return () => this._voiceVolumeSubscribers.delete(callback);
   };
 
+  protected _onVoiceConnected(): void {}
+
+  protected _onVoiceDisconnected(): void {}
+
+  protected _isRunActive(): boolean {
+    const runtime: ThreadRuntimeCore = this;
+    if (runtime.isRunning) return true;
+    const last = this._getBaseMessages().at(-1);
+    return (
+      last?.role === "assistant" &&
+      (last.status.type === "running" || last.status.type === "requires-action")
+    );
+  }
+
   public connectVoice() {
     const adapter = this.adapters?.voice;
     if (!adapter) throw new Error("Voice adapter not configured");
+    if (this._isRunActive())
+      throw new Error(
+        "Cannot start a voice session while a run is in progress or paused on a pending tool action",
+      );
+    const replacing = this._voiceSession !== undefined;
 
     try {
-      this.disconnectVoice();
+      this._disconnectVoice(false);
     } catch (error) {
       console.error(
         "[assistant-ui] Voice cleanup threw before reconnect",
@@ -397,7 +434,14 @@ export abstract class BaseThreadRuntimeCore
       );
     }
 
-    const session = adapter.connect({});
+    let session: RealtimeVoiceAdapter.Session;
+    try {
+      session = adapter.connect({});
+    } catch (error) {
+      if (replacing && this._voiceSession === undefined)
+        this._onVoiceDisconnected();
+      throw error;
+    }
     this._voiceSession = session;
     const unsubs: Array<() => void> = [];
     this._voiceUnsubs = unsubs;
@@ -433,10 +477,12 @@ export abstract class BaseThreadRuntimeCore
 
       unsubs.push(
         session.onStatusChange((status) => {
+          if (this._voiceSession !== session) return;
           if (status.type === "ended") {
             this._finishVoiceAssistantMessage();
             this._voiceSession = undefined;
             this.voice = undefined;
+            this._onVoiceDisconnected();
           } else {
             this.voice = {
               status,
@@ -477,17 +523,19 @@ export abstract class BaseThreadRuntimeCore
           this._handleVoiceTranscript(transcript);
         }),
       );
-      finishDetachedSetup();
+      if (!finishDetachedSetup()) this._onVoiceConnected();
     } catch (error) {
       if (this._voiceSession === session && this._voiceUnsubs === unsubs) {
         try {
-          this.disconnectVoice();
+          this._disconnectVoice(false);
         } catch (cleanupError) {
           console.error(
             "[assistant-ui] Voice rollback cleanup threw",
             cleanupError,
           );
         }
+        if (replacing && this._voiceSession === undefined)
+          this._onVoiceDisconnected();
       } else {
         finishDetachedSetup();
       }
@@ -507,7 +555,7 @@ export abstract class BaseThreadRuntimeCore
       this._currentAssistantMsg = null;
 
       if (transcript.isFinal) {
-        this._voiceMessages.push({
+        const message: ThreadMessage = {
           id: generateId(),
           role: "user",
           content: [{ type: "text", text: transcript.text }],
@@ -515,7 +563,9 @@ export abstract class BaseThreadRuntimeCore
           createdAt: new Date(),
           status: { type: "complete", reason: "unknown" },
           attachments: [],
-        });
+        };
+        this._voiceMessages.push(message);
+        this._commitVoiceMessage(message);
         this._markVoiceMessagesDirty();
         this._notifySubscribers();
       }
@@ -554,6 +604,7 @@ export abstract class BaseThreadRuntimeCore
       }
 
       if (transcript.isFinal) {
+        this._commitVoiceMessage(this._currentAssistantMsg);
         this._currentAssistantMsg = null;
       }
 
@@ -562,7 +613,7 @@ export abstract class BaseThreadRuntimeCore
     }
   }
 
-  private _finishVoiceAssistantMessage() {
+  private _finishVoiceAssistantMessage(notify = true) {
     const last = this._voiceMessages.at(-1);
     if (last?.role === "assistant" && last.status.type === "running") {
       const idx = this._voiceMessages.length - 1;
@@ -570,12 +621,19 @@ export abstract class BaseThreadRuntimeCore
         ...(last as ThreadAssistantMessage),
         status: { type: "complete", reason: "stop" },
       };
+      this._commitVoiceMessage(this._voiceMessages[idx]!);
+      this._currentAssistantMsg = null;
       this._markVoiceMessagesDirty();
-      this._notifySubscribers();
+      if (notify) this._notifySubscribers();
     }
   }
 
   public disconnectVoice() {
+    this._disconnectVoice(true);
+  }
+
+  private _disconnectVoice(fireHook: boolean) {
+    this._finishVoiceAssistantMessage(false);
     this._currentAssistantMsg = null;
     // Drain the shared list in place so reentrant setup cannot release the same handles again.
     const unsubs = this._voiceUnsubs.splice(0);
@@ -591,18 +649,23 @@ export abstract class BaseThreadRuntimeCore
     this._voiceMessages = [];
     this._markVoiceMessagesDirty();
 
-    notifySubscribers([
-      ...unsubs,
-      ...(stopSpeaking ? [stopSpeaking] : []),
-      ...(session ? [() => session.disconnect()] : []),
-      () =>
-        notifyEventListeners(
-          this._voiceVolumeSubscribers,
-          undefined,
-          "Voice volume",
-        ),
-      () => this._notifySubscribers(),
-    ]);
+    try {
+      notifySubscribers([
+        ...unsubs,
+        ...(stopSpeaking ? [stopSpeaking] : []),
+        ...(session ? [() => session.disconnect()] : []),
+        () =>
+          notifyEventListeners(
+            this._voiceVolumeSubscribers,
+            undefined,
+            "Voice volume",
+          ),
+        () => this._notifySubscribers(),
+      ]);
+    } finally {
+      if (fireHook && session && this._voiceSession === undefined)
+        this._onVoiceDisconnected();
+    }
   }
 
   public muteVoice() {

package/src/runtime/base/default-edit-composer-runtime-core.ts

@@ -14,7 +14,7 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   }
 
   public get canSend() {
-    return !this.isEmpty && !this._isSending;
+    return !this.isEmpty && !this.runtime.voice && !this._isSending;
   }
 
   protected getAttachmentAdapter() {
@@ -52,7 +52,17 @@ export class DefaultEditComposerRuntimeCore extends BaseComposerRuntimeCore {
   ) {
     super();
     this.runtime = runtime;
-    this.endEditCallback = endEditCallback;
+    let lastHasVoice = runtime.voice !== undefined;
+    const unsubscribe = runtime.subscribe(() => {
+      const hasVoice = runtime.voice !== undefined;
+      if (hasVoice === lastHasVoice) return;
+      lastHasVoice = hasVoice;
+      this._notifySubscribers();
+    });
+    this.endEditCallback = () => {
+      unsubscribe();
+      endEditCallback();
+    };
     this._parentId = parentId;
     this._sourceId = message.id;
     this.setText(getThreadMessageText(message));

package/src/runtime/base/default-thread-composer-runtime-core.ts

@@ -26,7 +26,12 @@ export class DefaultThreadComposerRuntimeCore
   }
 
   public get canSend() {
-    return !this.isEmpty && !this.runtime.isSendDisabled && !this._isSending;
+    return (
+      !this.isEmpty &&
+      !this.runtime.isSendDisabled &&
+      !this.runtime.voice &&
+      !this._isSending
+    );
   }
 
   private _queueCache:
@@ -99,6 +104,7 @@ export class DefaultThreadComposerRuntimeCore
   public connect() {
     let lastCanCancel = false;
     let lastIsSendDisabled = this.runtime.isSendDisabled;
+    let lastHasVoice = this.runtime.voice !== undefined;
     let lastQueue = this.queue;
     return this.runtime.subscribe(() => {
       let changed = false;
@@ -111,6 +117,11 @@ export class DefaultThreadComposerRuntimeCore
         lastIsSendDisabled = this.runtime.isSendDisabled;
         changed = true;
       }
+      const hasVoice = this.runtime.voice !== undefined;
+      if (lastHasVoice !== hasVoice) {
+        lastHasVoice = hasVoice;
+        changed = true;
+      }
       if (lastQueue !== this.queue) {
         lastQueue = this.queue;
         changed = true;

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -71,6 +71,7 @@ export type RespondToToolApprovalOptions = {
 export type SubmitFeedbackOptions = {
   messageId: string;
   type: "negative" | "positive";
+  comment?: string;
 };
 
 export type ThreadSuggestion = {
@@ -95,6 +96,7 @@ export type VoiceSessionState = {
 
 export type SubmittedFeedback = {
   readonly type: "negative" | "positive";
+  readonly comment?: string;
 };
 
 export type ThreadRuntimeEventPayload = {

package/src/runtime/queue/message-queue.ts

@@ -25,6 +25,8 @@ export type MessageQueueDriver = {
 
 export type MessageQueueController = {
   readonly adapter: ExternalThreadQueueAdapter;
+  hold: () => void;
+  release: () => void;
   /** Mark a run as in flight so concurrent sends buffer; call on the rising edge. */
   notifyBusy: () => void;
   /** Advances to the next pending message; call on the run's falling edge. */
@@ -88,6 +90,7 @@ export const createMessageQueue = (
 
   let running = false;
   let paused = false;
+  let held = false;
   let dispatchTransform: (message: AppendMessage) => AppendMessage = (m) => m;
   // swallow the cancelled run's settle when steering so it does not double-advance
   let suppressIdle = 0;
@@ -133,7 +136,7 @@ export const createMessageQueue = (
   };
 
   const advance = () => {
-    if (running || paused) return;
+    if (running || paused || held) return;
     const lane: Lane = lanes.steer.length > 0 ? "steer" : "queue";
     const head = lanes[lane][0];
     if (!head) return;
@@ -334,6 +337,13 @@ export const createMessageQueue = (
 
   return {
     adapter,
+    hold: () => {
+      held = true;
+    },
+    release: () => {
+      held = false;
+      advance();
+    },
     notifyBusy: () => {
       paused = false;
       // a cancelled run's settle that is still outstanding belongs to a run

package/src/runtime/utils/external-message-conversion.test.ts

@@ -152,6 +152,49 @@ describe("chunkExternalMessages", () => {
     ]);
     expect(chunks[1]?.inputs).toEqual([answer]);
   });
+
+  it("keeps a voice assistant transcript out of the neighbouring assistant chunks", () => {
+    const typed = {};
+    const spoken = {};
+    const spokenAgain = {};
+    const typedAgain = {};
+    const callbackResults: ExternalMessageConverterCallbackResult<object>[] = [
+      { input: typed, outputs: [{ role: "assistant", content: "typed" }] },
+      {
+        input: spoken,
+        outputs: [
+          {
+            role: "assistant",
+            content: "spoken",
+            metadata: { modality: "voice" },
+          },
+        ],
+      },
+      {
+        input: spokenAgain,
+        outputs: [
+          {
+            role: "assistant",
+            content: "spoken again",
+            metadata: { modality: "voice" },
+          },
+        ],
+      },
+      {
+        input: typedAgain,
+        outputs: [{ role: "assistant", content: "typed again" }],
+      },
+    ];
+
+    const chunks = chunkExternalMessages(callbackResults);
+
+    expect(chunks.map((chunk) => chunk.inputs)).toEqual([
+      [typed],
+      [spoken],
+      [spokenAgain],
+      [typedAgain],
+    ]);
+  });
 });
 
 describe("convertExternalMessageChunk", () => {

package/src/runtime/utils/external-message-conversion.ts

@@ -348,11 +348,14 @@ export const chunkExternalMessages = <T>(
 
   for (const callbackResult of callbackResults) {
     for (const output of callbackResult.outputs) {
+      const isVoice =
+        output.role === "assistant" && output.metadata?.modality === "voice";
       if (
         (pendingNone && output.role !== "tool") ||
         !isAssistant ||
         output.role === "user" ||
-        output.role === "system"
+        output.role === "system" ||
+        isVoice
       ) {
         flush();
       }
@@ -365,7 +368,8 @@ export const chunkExternalMessages = <T>(
 
       if (
         output.role === "assistant" &&
-        (output.convertConfig?.joinStrategy === "none" ||
+        (isVoice ||
+          output.convertConfig?.joinStrategy === "none" ||
           joinStrategy === "none")
       ) {
         pendingNone = true;

package/src/runtime/utils/stored-message-parts.ts

@@ -0,0 +1,174 @@
+import type { CompleteAttachment } from "../../types/attachment";
+import type {
+  ThreadAssistantMessagePart,
+  ThreadMessage,
+  ThreadUserMessagePart,
+} from "../../types/message";
+import { isRecord } from "../../utils/json/is-json";
+
+export const MAX_STORED_MESSAGE_DEPTH = 100;
+
+type StoredPartGuard = (part: Record<string, unknown>) => boolean;
+
+export type StoredMessagePart = Record<string, unknown> & { type: string };
+
+export const parseStoredDate = (value: unknown): Date | null => {
+  if (value instanceof Date && !Number.isNaN(value.getTime())) return value;
+  if (typeof value !== "string" && typeof value !== "number") return null;
+
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? null : date;
+};
+
+export const isStoredMessageRole = (
+  value: unknown,
+): value is ThreadMessage["role"] =>
+  value === "system" || value === "user" || value === "assistant";
+
+/**
+ * Builds the readability predicate for a persistence boundary. A stored part is
+ * readable when it passes the guard for its known type, or when its type is
+ * unknown and the boundary can still read it.
+ *
+ * Unknown types are where the boundaries part: local storage hands its rows to
+ * the runtime untouched, so a type written by a newer release still loads,
+ * while the aui/v0 decoder converts a `data-` prefix and throws on anything
+ * else unknown. Per-type differences ride in `overrides`.
+ */
+const makeIsStoredMessagePart = (
+  overrides: Partial<
+    Record<
+      (ThreadUserMessagePart | ThreadAssistantMessagePart)["type"],
+      StoredPartGuard
+    >
+  >,
+  isReadableUnknownType: (type: string) => boolean,
+) => {
+  const storedPartGuards = {
+    text: (part) => typeof part.text === "string",
+    reasoning: (part) =>
+      typeof part.text === "string" ||
+      typeof part.unstable_summary === "string",
+    image: (part) => typeof part.image === "string",
+    file: (part) =>
+      typeof part.data === "string" && typeof part.mimeType === "string",
+    audio: (part) =>
+      isRecord(part.audio) &&
+      typeof part.audio.data === "string" &&
+      typeof part.audio.format === "string",
+    data: (part) => typeof part.name === "string",
+    source: (part) =>
+      typeof part.id === "string" &&
+      (part.sourceType === "url"
+        ? typeof part.url === "string"
+        : part.sourceType === "document" &&
+          typeof part.title === "string" &&
+          typeof part.mediaType === "string"),
+    "generative-ui": (part) => isRecord(part.spec),
+    "tool-call": (part) =>
+      typeof part.toolCallId === "string" &&
+      typeof part.toolName === "string" &&
+      isRecord(part.args) &&
+      typeof part.argsText === "string",
+  } satisfies Record<
+    (ThreadUserMessagePart | ThreadAssistantMessagePart)["type"],
+    StoredPartGuard
+  >;
+  const guards: Record<string, StoredPartGuard> = {
+    ...storedPartGuards,
+    ...overrides,
+  };
+
+  return (value: unknown): value is StoredMessagePart =>
+    isRecord(value) &&
+    typeof value.type === "string" &&
+    (Object.hasOwn(guards, value.type)
+      ? guards[value.type]!(value)
+      : isReadableUnknownType(value.type));
+};
+
+export const isStoredMessagePart = makeIsStoredMessagePart({}, () => true);
+
+/**
+ * The aui/v0 decoder dereferences each present field rather than reading only
+ * the one it needs, so this boundary checks every field it will touch, while
+ * local storage stays no stricter than what the runtime itself writes.
+ */
+const isStoredAuiV0Part = makeIsStoredMessagePart(
+  {
+    reasoning: (part) =>
+      (part.text === undefined || typeof part.text === "string") &&
+      (part.unstable_summary === undefined ||
+        typeof part.unstable_summary === "string") &&
+      (typeof part.text === "string" ||
+        typeof part.unstable_summary === "string"),
+    "tool-call": (part) =>
+      typeof part.toolCallId === "string" &&
+      typeof part.toolName === "string" &&
+      (part.args === undefined || isRecord(part.args)) &&
+      (part.argsText === undefined || typeof part.argsText === "string") &&
+      (part.args !== undefined || part.argsText !== undefined),
+  },
+  (type) => type.startsWith("data-"),
+);
+
+const rolePartTypes = {
+  user: {
+    text: true,
+    image: true,
+    file: true,
+    data: true,
+    audio: true,
+  } satisfies Record<ThreadUserMessagePart["type"], true>,
+  assistant: {
+    text: true,
+    reasoning: true,
+    "tool-call": true,
+    source: true,
+    file: true,
+    image: true,
+    data: true,
+    "generative-ui": true,
+  } satisfies Record<ThreadAssistantMessagePart["type"], true>,
+};
+
+/**
+ * `fromThreadMessageLike` converts each part through its role's branch and
+ * throws on a part that branch does not carry, which would cost the whole
+ * stored row. The aui/v0 boundary decodes through that converter, so it filters
+ * by role as well as by shape. A part type unknown to this table is left to the
+ * boundary predicate, which keeps the `data-` prefixes the converter handles.
+ */
+export const isStoredAuiV0RolePart = (
+  role: ThreadMessage["role"],
+  value: unknown,
+): value is StoredMessagePart =>
+  isStoredAuiV0Part(value) &&
+  (role === "system"
+    ? value.type === "text"
+    : Object.hasOwn(rolePartTypes.assistant, value.type) ||
+        Object.hasOwn(rolePartTypes.user, value.type)
+      ? Object.hasOwn(rolePartTypes[role], value.type)
+      : true);
+
+export const parseStoredAttachment = (
+  value: unknown,
+  isPart: (value: unknown) => value is StoredMessagePart,
+): CompleteAttachment | null => {
+  if (
+    !isRecord(value) ||
+    typeof value.id !== "string" ||
+    typeof value.type !== "string" ||
+    typeof value.name !== "string" ||
+    !isRecord(value.status) ||
+    value.status.type !== "complete" ||
+    !Array.isArray(value.content)
+  ) {
+    return null;
+  }
+
+  return {
+    ...value,
+    content: value.content.filter(isPart),
+  } as CompleteAttachment;
+};

package/src/runtime/utils/thread-message-like.ts

@@ -80,14 +80,19 @@ export type ThreadMessageLike = {
     | undefined;
   readonly metadata?:
     | {
-        readonly unstable_state?: ReadonlyJSONValue;
+        readonly unstable_state?: ReadonlyJSONValue | undefined;
         readonly unstable_annotations?:
           | readonly ReadonlyJSONValue[]
           | undefined;
         readonly unstable_data?: readonly ReadonlyJSONValue[] | undefined;
         readonly steps?: readonly ThreadStep[] | undefined;
         readonly timing?: MessageTiming | undefined;
-        readonly submittedFeedback?: { readonly type: "positive" | "negative" };
+        readonly submittedFeedback?:
+          | {
+              readonly type: "positive" | "negative";
+              readonly comment?: string;
+            }
+          | undefined;
         readonly isOptimistic?: boolean | undefined;
         readonly modality?: MessageModality | undefined;
         readonly custom?: Record<string, unknown> | undefined;

package/src/runtimes/external-store/external-store-adapter.ts

@@ -133,6 +133,10 @@ type ExternalStoreAdapterBase<T> = {
    * back, because the runtime cannot see a removal it did not make.
    */
   setMessages?: ((messages: readonly T[]) => void) | undefined;
+  /**
+   * Called with each finalized voice transcript message. The host appends it to its own messages under the same id, which is how the runtime knows the host carries it. A host that does not implement this callback keeps transcripts for the session only.
+   */
+  onVoiceTranscript?: ((message: ThreadMessage) => void) | undefined;
   /**
    * Fires when the user explicitly switches branches via the runtime's
    * `switchToBranch` action (e.g. a BranchPicker click). It does not fire on