@assistant-ui/core 0.1.8 → 0.1.10

package/src/runtime/base/base-thread-runtime-core.ts

@@ -1,7 +1,12 @@
-import type { AppendMessage, ThreadMessage } from "../../types/message";
+import type {
+  AppendMessage,
+  ThreadAssistantMessage,
+  ThreadMessage,
+} from "../../types/message";
 import type { Unsubscribe } from "../../types/unsubscribe";
 import type { ModelContextProvider } from "../../model-context/types";
 import { getThreadMessageText } from "../../utils/text";
+import { generateId } from "../../utils/id";
 import {
   ExportedMessageRepository,
   MessageRepository,
@@ -14,6 +19,7 @@ import type {
   SubmitFeedbackOptions,
   ThreadRuntimeCore,
   SpeechState,
+  VoiceSessionState,
   RuntimeCapabilities,
   ThreadRuntimeEventType,
   StartRunConfig,
@@ -23,12 +29,14 @@ import { DefaultEditComposerRuntimeCore } from "./default-edit-composer-runtime-
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
 import type { FeedbackAdapter } from "../../adapters/feedback";
 import type { AttachmentAdapter } from "../../adapters/attachment";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 
 type BaseThreadAdapters = {
   speech?: SpeechSynthesisAdapter | undefined;
   feedback?: FeedbackAdapter | undefined;
   attachments?: AttachmentAdapter | undefined;
+  voice?: RealtimeVoiceAdapter | undefined;
 };
 
 export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
@@ -53,10 +61,37 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
   public abstract importExternalState(state: any): void;
   public abstract unstable_loadExternalState(state: any): void;
 
-  public get messages() {
+  protected _voiceMessages: ThreadMessage[] = [];
+  protected _voiceGeneration = 0;
+  private _cachedMergedMessages: readonly ThreadMessage[] | null = null;
+  private _cachedVoiceGeneration = -1;
+  private _cachedMergedBase: readonly ThreadMessage[] | null = null;
+
+  protected _markVoiceMessagesDirty() {
+    this._voiceGeneration++;
+    this._cachedMergedMessages = null;
+  }
+
+  protected _getBaseMessages(): readonly ThreadMessage[] {
     return this.repository.getMessages();
   }
 
+  public get messages(): readonly ThreadMessage[] {
+    if (this._voiceMessages.length === 0) {
+      return this._getBaseMessages();
+    }
+    const base = this._getBaseMessages();
+    if (
+      this._cachedVoiceGeneration !== this._voiceGeneration ||
+      this._cachedMergedBase !== base
+    ) {
+      this._cachedMergedMessages = [...base, ...this._voiceMessages];
+      this._cachedVoiceGeneration = this._voiceGeneration;
+      this._cachedMergedBase = base;
+    }
+    return this._cachedMergedMessages!;
+  }
+
   public get state() {
     let mostRecentAssistantMessage;
     for (const message of this.messages) {
@@ -99,11 +134,28 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
     try {
       return this.repository.getMessage(messageId);
     } catch {
+      // Check voice messages
+      const baseMessages = this.repository.getMessages();
+      const voiceIdx = this._voiceMessages.findIndex((m) => m.id === messageId);
+      if (voiceIdx !== -1) {
+        const parentId =
+          voiceIdx > 0
+            ? this._voiceMessages[voiceIdx - 1]!.id
+            : (baseMessages.at(-1)?.id ?? null);
+        return {
+          parentId,
+          message: this._voiceMessages[voiceIdx]!,
+          index: baseMessages.length + voiceIdx,
+        };
+      }
       return undefined;
     }
   }
 
   public getBranches(messageId: string): string[] {
+    if (this._voiceMessages.some((m) => m.id === messageId)) {
+      return [];
+    }
     return this.repository.getBranches(messageId);
   }
 
@@ -188,6 +240,195 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
     this._notifySubscribers();
   }
 
+  private _voiceSession: RealtimeVoiceAdapter.Session | undefined;
+  private _voiceUnsubs: Array<() => void> = [];
+  public voice: VoiceSessionState | undefined;
+
+  private _voiceVolume = 0;
+  private _voiceVolumeSubscribers = new Set<() => void>();
+
+  public getVoiceVolume = () => this._voiceVolume;
+
+  public subscribeVoiceVolume = (callback: () => void): Unsubscribe => {
+    this._voiceVolumeSubscribers.add(callback);
+    return () => this._voiceVolumeSubscribers.delete(callback);
+  };
+
+  public connectVoice() {
+    const adapter = this.adapters?.voice;
+    if (!adapter) throw new Error("Voice adapter not configured");
+
+    this.disconnectVoice();
+
+    const session = adapter.connect({});
+    this._voiceSession = session;
+    const unsubs: Array<() => void> = [];
+
+    let currentMode: RealtimeVoiceAdapter.Mode = "listening";
+
+    this.voice = {
+      status: session.status,
+      isMuted: session.isMuted,
+      mode: currentMode,
+    };
+    this._voiceVolume = 0;
+    this._notifySubscribers();
+
+    unsubs.push(
+      session.onStatusChange((status) => {
+        if (status.type === "ended") {
+          this._finishVoiceAssistantMessage();
+          this._voiceSession = undefined;
+          this.voice = undefined;
+        } else {
+          this.voice = {
+            status,
+            isMuted: session.isMuted,
+            mode: currentMode,
+          };
+        }
+        this._notifySubscribers();
+      }),
+    );
+
+    unsubs.push(
+      session.onModeChange((mode) => {
+        currentMode = mode;
+        if (this.voice) {
+          this.voice = { ...this.voice, mode };
+          this._notifySubscribers();
+        }
+      }),
+    );
+
+    unsubs.push(
+      session.onVolumeChange((volume) => {
+        this._voiceVolume = volume;
+        for (const cb of this._voiceVolumeSubscribers) cb();
+      }),
+    );
+
+    unsubs.push(
+      session.onTranscript((transcript) => {
+        this._handleVoiceTranscript(transcript);
+      }),
+    );
+
+    this._voiceUnsubs = unsubs;
+  }
+
+  private _currentAssistantMsg: ThreadAssistantMessage | null = null;
+
+  private _handleVoiceTranscript(
+    transcript: RealtimeVoiceAdapter.TranscriptItem,
+  ) {
+    this.ensureInitialized();
+
+    if (transcript.role === "user") {
+      this._finishVoiceAssistantMessage();
+      this._currentAssistantMsg = null;
+
+      if (transcript.isFinal) {
+        this._voiceMessages.push({
+          id: generateId(),
+          role: "user",
+          content: [{ type: "text", text: transcript.text }],
+          metadata: { custom: {} },
+          createdAt: new Date(),
+          status: { type: "complete", reason: "unknown" },
+          attachments: [],
+        });
+        this._markVoiceMessagesDirty();
+        this._notifySubscribers();
+      }
+    } else {
+      if (!this._currentAssistantMsg) {
+        this._currentAssistantMsg = {
+          id: generateId(),
+          role: "assistant",
+          content: [{ type: "text", text: transcript.text }],
+          metadata: {
+            unstable_state: this.state,
+            unstable_annotations: [],
+            unstable_data: [],
+            steps: [],
+            custom: {},
+          },
+          status: { type: "running" },
+          createdAt: new Date(),
+        };
+        this._voiceMessages.push(this._currentAssistantMsg);
+      } else {
+        const idx = this._voiceMessages.indexOf(this._currentAssistantMsg);
+        if (idx === -1) return;
+        const updated: ThreadAssistantMessage = {
+          ...this._currentAssistantMsg,
+          content: [{ type: "text", text: transcript.text }],
+          ...(transcript.isFinal
+            ? { status: { type: "complete", reason: "stop" } }
+            : {}),
+        };
+        this._voiceMessages[idx] = updated;
+        this._currentAssistantMsg = updated;
+      }
+
+      if (transcript.isFinal) {
+        this._currentAssistantMsg = null;
+      }
+
+      this._markVoiceMessagesDirty();
+      this._notifySubscribers();
+    }
+  }
+
+  private _finishVoiceAssistantMessage() {
+    const last = this._voiceMessages.at(-1);
+    if (last?.role === "assistant" && last.status.type === "running") {
+      const idx = this._voiceMessages.length - 1;
+      this._voiceMessages[idx] = {
+        ...(last as ThreadAssistantMessage),
+        status: { type: "complete", reason: "stop" },
+      };
+      this._markVoiceMessagesDirty();
+      this._notifySubscribers();
+    }
+  }
+
+  public disconnectVoice() {
+    this._finishVoiceAssistantMessage();
+    this._currentAssistantMsg = null;
+    for (const unsub of this._voiceUnsubs) unsub();
+    this._voiceUnsubs = [];
+    this._voiceSession?.disconnect();
+    this._voiceSession = undefined;
+    this.voice = undefined;
+    this._voiceVolume = 0;
+    for (const cb of this._voiceVolumeSubscribers) cb();
+    this._voiceMessages = [];
+    this._markVoiceMessagesDirty();
+    this._notifySubscribers();
+  }
+
+  public muteVoice() {
+    if (!this._voiceSession) throw new Error("No active voice session");
+    this._voiceSession.mute();
+    this.voice = {
+      ...this.voice!,
+      isMuted: true,
+    };
+    this._notifySubscribers();
+  }
+
+  public unmuteVoice() {
+    if (!this._voiceSession) throw new Error("No active voice session");
+    this._voiceSession.unmute();
+    this.voice = {
+      ...this.voice!,
+      isMuted: false,
+    };
+    this._notifySubscribers();
+  }
+
   protected ensureInitialized() {
     if (!this._isInitialized) {
       this._isInitialized = true;

package/src/runtime/interfaces/composer-runtime-core.ts

@@ -5,7 +5,10 @@ import type { Unsubscribe } from "../../types/unsubscribe";
 import type { RunConfig } from "../../types/message";
 import type { DictationAdapter } from "../../adapters/speech";
 
-export type ComposerRuntimeEventType = "send" | "attachmentAdd";
+export type ComposerRuntimeEventType =
+  | "send"
+  | "attachmentAdd"
+  | "attachmentAddError";
 
 export type DictationState = {
   readonly status: DictationAdapter.Status;

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -4,6 +4,7 @@ import type { Unsubscribe } from "../../types/unsubscribe";
 import type { AppendMessage, ThreadMessage } from "../../types/message";
 import type { RunConfig } from "../../types/message";
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type {
   ChatModelRunOptions,
   ChatModelRunResult,
@@ -24,6 +25,7 @@ export type RuntimeCapabilities = {
   readonly unstable_copy: boolean;
   readonly speech: boolean;
   readonly dictation: boolean;
+  readonly voice: boolean;
   readonly attachments: boolean;
   readonly feedback: boolean;
   readonly queue: boolean;
@@ -57,6 +59,12 @@ export type SpeechState = {
   readonly status: SpeechSynthesisAdapter.Status;
 };
 
+export type VoiceSessionState = {
+  readonly status: RealtimeVoiceAdapter.Status;
+  readonly isMuted: boolean;
+  readonly mode: RealtimeVoiceAdapter.Mode;
+};
+
 export type SubmittedFeedback = {
   readonly type: "negative" | "positive";
 };
@@ -102,6 +110,11 @@ export type ThreadRuntimeCore = Readonly<{
   speak: (messageId: string) => void;
   stopSpeaking: () => void;
 
+  connectVoice: () => void;
+  disconnectVoice: () => void;
+  muteVoice: () => void;
+  unmuteVoice: () => void;
+
   submitFeedback: (feedback: SubmitFeedbackOptions) => void;
 
   getModelContext: () => ModelContext;
@@ -111,6 +124,7 @@ export type ThreadRuntimeCore = Readonly<{
   beginEdit: (messageId: string) => void;
 
   speech: SpeechState | undefined;
+  voice: VoiceSessionState | undefined;
 
   capabilities: Readonly<RuntimeCapabilities>;
   isDisabled: boolean;
@@ -123,6 +137,9 @@ export type ThreadRuntimeCore = Readonly<{
 
   subscribe: (callback: () => void) => Unsubscribe;
 
+  getVoiceVolume: () => number;
+  subscribeVoiceVolume: (callback: () => void) => Unsubscribe;
+
   import(repository: ExportedMessageRepository): void;
   export(): ExportedMessageRepository;
 

package/src/runtimes/external-store/external-store-adapter.ts

@@ -5,6 +5,7 @@ import type {
   SpeechSynthesisAdapter,
   DictationAdapter,
 } from "../../adapters/speech";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { FeedbackAdapter } from "../../adapters/feedback";
 import type {
   AddToolResultOptions,
@@ -90,6 +91,7 @@ type ExternalStoreAdapterBase<T> = {
         attachments?: AttachmentAdapter | undefined;
         speech?: SpeechSynthesisAdapter | undefined;
         dictation?: DictationAdapter | undefined;
+        voice?: RealtimeVoiceAdapter | undefined;
         feedback?: FeedbackAdapter | undefined;
         /**
          * @deprecated This API is still under active development and might change without notice.

package/src/runtimes/external-store/external-store-thread-runtime-core.ts

@@ -63,6 +63,7 @@ export class ExternalStoreThreadRuntimeCore
     unstable_copy: false,
     speech: false,
     dictation: false,
+    voice: false,
     attachments: false,
     feedback: false,
     queue: false,
@@ -78,7 +79,7 @@ export class ExternalStoreThreadRuntimeCore
     return this._store.isLoading ?? false;
   }
 
-  public override get messages() {
+  protected override _getBaseMessages(): readonly ThreadMessage[] {
     return this._messages;
   }
 
@@ -137,6 +138,7 @@ export class ExternalStoreThreadRuntimeCore
       cancel: this._store.onCancel !== undefined,
       speech: this._store.adapters?.speech !== undefined,
       dictation: this._store.adapters?.dictation !== undefined,
+      voice: this._store.adapters?.voice !== undefined,
       unstable_copy: this._store.unstable_capabilities?.copy !== false,
       attachments: !!this._store.adapters?.attachments,
       feedback: !!this._store.adapters?.feedback,

package/src/runtimes/local/local-runtime-options.ts

@@ -5,6 +5,7 @@ import type {
   SpeechSynthesisAdapter,
   DictationAdapter,
 } from "../../adapters/speech";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { SuggestionAdapter } from "../../adapters/suggestion";
 import type { ChatModelAdapter } from "../../runtime/utils/chat-model-adapter";
 
@@ -16,6 +17,7 @@ export type LocalRuntimeOptionsBase = {
     attachments?: AttachmentAdapter | undefined;
     speech?: SpeechSynthesisAdapter | undefined;
     dictation?: DictationAdapter | undefined;
+    voice?: RealtimeVoiceAdapter | undefined;
     feedback?: FeedbackAdapter | undefined;
     suggestion?: SuggestionAdapter | undefined;
   };

package/src/runtimes/local/local-thread-runtime-core.ts

@@ -45,6 +45,7 @@ export class LocalThreadRuntimeCore
     unstable_copy: true,
     speech: false,
     dictation: false,
+    voice: false,
     attachments: false,
     feedback: false,
     queue: false,
@@ -118,6 +119,12 @@ export class LocalThreadRuntimeCore
       hasUpdates = true;
     }
 
+    const canVoice = options.adapters?.voice !== undefined;
+    if (this.capabilities.voice !== canVoice) {
+      this.capabilities.voice = canVoice;
+      hasUpdates = true;
+    }
+
     const canAttach = options.adapters?.attachments !== undefined;
     if (this.capabilities.attachments !== canAttach) {
       this.capabilities.attachments = canAttach;

package/src/runtimes/readonly/ReadonlyThreadRuntimeCore.ts

@@ -34,7 +34,6 @@ export class ReadonlyThreadRuntimeCore
   }
 
   getBranches(messageId: string) {
-    // no branching in readonly threads
     const idx = this._messages.findIndex((m) => m.id === messageId);
     if (idx === -1) return [];
     return [messageId];
@@ -56,9 +55,7 @@ export class ReadonlyThreadRuntimeCore
     throw READONLY_THREAD_ERROR;
   }
 
-  cancelRun(): void {
-    // noop - nothing to cancel
-  }
+  cancelRun(): void {}
 
   addToolResult(): void {
     throw READONLY_THREAD_ERROR;
@@ -72,8 +69,23 @@ export class ReadonlyThreadRuntimeCore
     throw READONLY_THREAD_ERROR;
   }
 
-  stopSpeaking(): void {
-    // noop
+  stopSpeaking(): void {}
+
+  connectVoice(): void {
+    throw READONLY_THREAD_ERROR;
+  }
+
+  disconnectVoice(): void {}
+
+  getVoiceVolume = () => 0;
+  subscribeVoiceVolume = (): Unsubscribe => () => {};
+
+  muteVoice(): void {
+    throw READONLY_THREAD_ERROR;
+  }
+
+  unmuteVoice(): void {
+    throw READONLY_THREAD_ERROR;
   }
 
   submitFeedback(): void {
@@ -179,6 +191,7 @@ export class ReadonlyThreadRuntimeCore
   }
 
   speech = undefined;
+  voice = undefined;
 
   capabilities = {
     switchToBranch: false,
@@ -189,6 +202,7 @@ export class ReadonlyThreadRuntimeCore
     unstable_copy: false,
     speech: false,
     dictation: false,
+    voice: false,
     attachments: false,
     feedback: false,
     queue: false,

package/src/runtimes/remote-thread-list/empty-thread-core.ts

@@ -48,6 +48,25 @@ export const EMPTY_THREAD_CORE: ThreadRuntimeCore = {
     throw EMPTY_THREAD_ERROR;
   },
 
+  connectVoice() {
+    throw EMPTY_THREAD_ERROR;
+  },
+
+  disconnectVoice() {
+    throw EMPTY_THREAD_ERROR;
+  },
+
+  getVoiceVolume: () => 0,
+  subscribeVoiceVolume: () => () => {},
+
+  muteVoice() {
+    throw EMPTY_THREAD_ERROR;
+  },
+
+  unmuteVoice() {
+    throw EMPTY_THREAD_ERROR;
+  },
+
   submitFeedback() {
     throw EMPTY_THREAD_ERROR;
   },
@@ -72,7 +91,7 @@ export const EMPTY_THREAD_CORE: ThreadRuntimeCore = {
     attachments: [],
     attachmentAccept: "*",
 
-    async addAttachment(_fileOrAttachment) {
+    async addAttachment() {
       throw EMPTY_THREAD_ERROR;
     },
 
@@ -153,6 +172,7 @@ export const EMPTY_THREAD_CORE: ThreadRuntimeCore = {
   },
 
   speech: undefined,
+  voice: undefined,
 
   capabilities: {
     switchToBranch: false,
@@ -163,6 +183,7 @@ export const EMPTY_THREAD_CORE: ThreadRuntimeCore = {
     unstable_copy: false,
     speech: false,
     dictation: false,
+    voice: false,
     attachments: false,
     feedback: false,
     queue: false,

package/src/store/runtime-clients/composer-runtime-client.ts

@@ -16,7 +16,6 @@ import {
   ComposerRuntime,
   EditComposerRuntime,
 } from "../../runtime/api/composer-runtime";
-import { ComposerRuntimeEventType } from "../../runtime/interfaces/composer-runtime-core";
 import { ComposerState } from "../scopes/composer";
 import { AttachmentRuntimeClient } from "./attachment-runtime-client";
 import { tapSubscribable } from "./tap-subscribable";
@@ -54,12 +53,7 @@ export const ComposerClient = resource(
       const unsubscribers: Unsubscribe[] = [];
 
       // Subscribe to composer events
-      const composerEvents: ComposerRuntimeEventType[] = [
-        "send",
-        "attachmentAdd",
-      ];
-
-      for (const event of composerEvents) {
+      for (const event of ["send", "attachmentAdd"] as const) {
         const unsubscribe = runtime.unstable_on(event, () => {
           emit(`composer.${event}`, {
             threadId: threadIdRef.current,
@@ -69,6 +63,23 @@ export const ComposerClient = resource(
         unsubscribers.push(unsubscribe);
       }
 
+      // attachmentAddError carries the failed attachment ID
+      unsubscribers.push(
+        runtime.unstable_on("attachmentAddError", () => {
+          const errorAttachment = runtime
+            .getState()
+            .attachments.findLast(
+              (a) =>
+                a.status.type === "incomplete" && a.status.reason === "error",
+            );
+          emit("composer.attachmentAddError", {
+            threadId: threadIdRef.current,
+            ...(messageIdRef && { messageId: messageIdRef.current }),
+            ...(errorAttachment && { attachmentId: errorAttachment.id }),
+          });
+        }),
+      );
+
       return () => {
         for (const unsub of unsubscribers) unsub();
       };

package/src/store/runtime-clients/thread-runtime-client.ts

@@ -44,11 +44,9 @@ export const ThreadClient = resource(
     const runtimeState = tapSubscribable(runtime);
     const emit = tapAssistantEmit();
 
-    // Bind thread events to event manager
     tapEffect(() => {
       const unsubscribers: Unsubscribe[] = [];
 
-      // Subscribe to thread events
       const threadEvents: ThreadRuntimeEventType[] = [
         "runStart",
         "runEnd",
@@ -105,6 +103,7 @@ export const ThreadClient = resource(
         suggestions: runtimeState.suggestions,
         extras: runtimeState.extras,
         speech: runtimeState.speech,
+        voice: runtimeState.voice,
 
         composer: composer.state,
         messages: messages.state,
@@ -124,12 +123,12 @@ export const ThreadClient = resource(
       import: runtime.import,
       reset: runtime.reset,
       stopSpeaking: runtime.stopSpeaking,
-      startVoice: async () => {
-        throw new Error("startVoice is not supported in this runtime");
-      },
-      stopVoice: async () => {
-        throw new Error("stopVoice is not supported in this runtime");
-      },
+      connectVoice: runtime.connectVoice,
+      disconnectVoice: runtime.disconnectVoice,
+      getVoiceVolume: runtime.getVoiceVolume,
+      subscribeVoiceVolume: runtime.subscribeVoiceVolume,
+      muteVoice: runtime.muteVoice,
+      unmuteVoice: runtime.unmuteVoice,
       message: (selector) => {
         if ("id" in selector) {
           return messages.get({ key: selector.id });

package/src/store/scopes/composer.ts

@@ -90,6 +90,11 @@ export type ComposerMeta = {
 export type ComposerEvents = {
   "composer.send": { threadId: string; messageId?: string };
   "composer.attachmentAdd": { threadId: string; messageId?: string };
+  "composer.attachmentAddError": {
+    threadId: string;
+    messageId?: string;
+    attachmentId?: string;
+  };
 };
 
 export type ComposerClientSchema = {