@assistant-ui/core 0.1.9 → 0.1.10

package/src/react/model-context/useToolArgsStatus.ts

@@ -0,0 +1,51 @@
+import { useMemo } from "react";
+import { useAuiState } from "@assistant-ui/store";
+import {
+  getPartialJsonObjectFieldState,
+  getPartialJsonObjectMeta,
+} from "assistant-stream/utils";
+
+type PropFieldStatus = "streaming" | "complete";
+
+export type ToolArgsStatus<
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  status: "running" | "complete" | "incomplete" | "requires-action";
+  propStatus: Partial<Record<keyof TArgs, PropFieldStatus>>;
+};
+
+export const useToolArgsStatus = <
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+>(): ToolArgsStatus<TArgs> => {
+  const part = useAuiState((s) => s.part);
+
+  return useMemo(() => {
+    const statusType = part.status.type;
+
+    if (part.type !== "tool-call") {
+      throw new Error(
+        "useToolArgsStatus can only be used inside tool-call message parts",
+      );
+    }
+
+    const isStreaming = statusType === "running";
+    const args = part.args as Record<string, unknown>;
+    const meta = getPartialJsonObjectMeta(args as Record<symbol, unknown>);
+    const propStatus: Partial<Record<string, PropFieldStatus>> = {};
+
+    for (const key of Object.keys(args)) {
+      if (meta) {
+        const fieldState = getPartialJsonObjectFieldState(args, [key]);
+        propStatus[key] =
+          fieldState === "complete" || !isStreaming ? "complete" : "streaming";
+      } else {
+        propStatus[key] = isStreaming ? "streaming" : "complete";
+      }
+    }
+
+    return {
+      status: statusType,
+      propStatus: propStatus as Partial<Record<keyof TArgs, PropFieldStatus>>,
+    };
+  }, [part]);
+};

package/src/react/primitive-hooks/useVoice.ts

@@ -0,0 +1,41 @@
+import { useCallback, useSyncExternalStore } from "react";
+import { useAui, useAuiState } from "@assistant-ui/store";
+import type { VoiceSessionState } from "../../runtime/interfaces/thread-runtime-core";
+
+export const useVoiceState = (): VoiceSessionState | undefined => {
+  return useAuiState((s) => s.thread.voice);
+};
+
+const getServerVolume = () => 0;
+
+export const useVoiceVolume = (): number => {
+  const aui = useAui();
+  const thread = aui.thread();
+  return useSyncExternalStore(
+    thread.subscribeVoiceVolume,
+    thread.getVoiceVolume,
+    getServerVolume,
+  );
+};
+
+export const useVoiceControls = () => {
+  const aui = useAui();
+
+  const connect = useCallback(() => {
+    aui.thread().connectVoice();
+  }, [aui]);
+
+  const disconnect = useCallback(() => {
+    aui.thread().disconnectVoice();
+  }, [aui]);
+
+  const mute = useCallback(() => {
+    aui.thread().muteVoice();
+  }, [aui]);
+
+  const unmute = useCallback(() => {
+    aui.thread().unmuteVoice();
+  }, [aui]);
+
+  return { connect, disconnect, mute, unmute };
+};

package/src/react/types/scopes/interactables.ts

@@ -27,9 +27,25 @@ export type InteractableRegistration = {
   selected?: boolean | undefined;
 };
 
+export type InteractablePersistenceStatus = {
+  isPending: boolean;
+  error: unknown;
+};
+
 export type InteractablesState = {
   /** Keyed by instance id */
   definitions: Record<string, InteractableDefinition>;
+  /** Per-id persistence sync status */
+  persistence: Record<string, InteractablePersistenceStatus>;
+};
+
+export type InteractablePersistedState = Record<
+  string,
+  { name: string; state: unknown }
+>;
+
+export type InteractablePersistenceAdapter = {
+  save(state: InteractablePersistedState): void | Promise<void>;
 };
 
 export type InteractablesMethods = {
@@ -37,6 +53,12 @@ export type InteractablesMethods = {
   register(def: InteractableRegistration): Unsubscribe;
   setState(id: string, updater: (prev: unknown) => unknown): void;
   setSelected(id: string, selected: boolean): void;
+  exportState(): InteractablePersistedState;
+  importState(saved: InteractablePersistedState): void;
+  setPersistenceAdapter(
+    adapter: InteractablePersistenceAdapter | undefined,
+  ): void;
+  flush(): Promise<void>;
 };
 
 export type InteractablesClientSchema = {

package/src/runtime/api/thread-runtime.ts

@@ -3,6 +3,7 @@ import {
   RuntimeCapabilities,
   ThreadRuntimeCore,
   SpeechState,
+  VoiceSessionState,
   ThreadRuntimeEventType,
   StartRunConfig,
   ResumeRunConfig,
@@ -187,6 +188,8 @@ export type ThreadState = {
    * @deprecated This API is still under active development and might change without notice.
    */
   readonly speech: SpeechState | undefined;
+
+  readonly voice: VoiceSessionState | undefined;
 };
 
 export const getThreadState = (
@@ -209,6 +212,7 @@ export const getThreadState = (
     suggestions: runtime.suggestions,
     extras: runtime.extras,
     speech: runtime.speech,
+    voice: runtime.voice,
   });
 };
 
@@ -317,6 +321,13 @@ export type ThreadRuntime = {
    */
   stopSpeaking(): void;
 
+  connectVoice(): void;
+  disconnectVoice(): void;
+  getVoiceVolume(): number;
+  subscribeVoiceVolume(callback: () => void): Unsubscribe;
+  muteVoice(): void;
+  unmuteVoice(): void;
+
   unstable_on(event: ThreadRuntimeEventType, callback: () => void): Unsubscribe;
 };
 
@@ -388,6 +399,12 @@ export class ThreadRuntimeImpl implements ThreadRuntime {
     this.startRun = this.startRun.bind(this);
     this.cancelRun = this.cancelRun.bind(this);
     this.stopSpeaking = this.stopSpeaking.bind(this);
+    this.connectVoice = this.connectVoice.bind(this);
+    this.disconnectVoice = this.disconnectVoice.bind(this);
+    this.muteVoice = this.muteVoice.bind(this);
+    this.unmuteVoice = this.unmuteVoice.bind(this);
+    this.getVoiceVolume = this.getVoiceVolume.bind(this);
+    this.subscribeVoiceVolume = this.subscribeVoiceVolume.bind(this);
     this.export = this.export.bind(this);
     this.import = this.import.bind(this);
     this.reset = this.reset.bind(this);
@@ -463,6 +480,30 @@ export class ThreadRuntimeImpl implements ThreadRuntime {
     return this._threadBinding.getState().stopSpeaking();
   }
 
+  public connectVoice() {
+    this._threadBinding.getState().connectVoice();
+  }
+
+  public disconnectVoice() {
+    this._threadBinding.getState().disconnectVoice();
+  }
+
+  public getVoiceVolume() {
+    return this._threadBinding.getState().getVoiceVolume();
+  }
+
+  public subscribeVoiceVolume(callback: () => void) {
+    return this._threadBinding.getState().subscribeVoiceVolume(callback);
+  }
+
+  public muteVoice() {
+    this._threadBinding.getState().muteVoice();
+  }
+
+  public unmuteVoice() {
+    this._threadBinding.getState().unmuteVoice();
+  }
+
   public export() {
     return this._threadBinding.getState().export();
   }

package/src/runtime/base/base-thread-runtime-core.ts

@@ -1,7 +1,12 @@
-import type { AppendMessage, ThreadMessage } from "../../types/message";
+import type {
+  AppendMessage,
+  ThreadAssistantMessage,
+  ThreadMessage,
+} from "../../types/message";
 import type { Unsubscribe } from "../../types/unsubscribe";
 import type { ModelContextProvider } from "../../model-context/types";
 import { getThreadMessageText } from "../../utils/text";
+import { generateId } from "../../utils/id";
 import {
   ExportedMessageRepository,
   MessageRepository,
@@ -14,6 +19,7 @@ import type {
   SubmitFeedbackOptions,
   ThreadRuntimeCore,
   SpeechState,
+  VoiceSessionState,
   RuntimeCapabilities,
   ThreadRuntimeEventType,
   StartRunConfig,
@@ -23,12 +29,14 @@ import { DefaultEditComposerRuntimeCore } from "./default-edit-composer-runtime-
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
 import type { FeedbackAdapter } from "../../adapters/feedback";
 import type { AttachmentAdapter } from "../../adapters/attachment";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 
 type BaseThreadAdapters = {
   speech?: SpeechSynthesisAdapter | undefined;
   feedback?: FeedbackAdapter | undefined;
   attachments?: AttachmentAdapter | undefined;
+  voice?: RealtimeVoiceAdapter | undefined;
 };
 
 export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
@@ -53,10 +61,37 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
   public abstract importExternalState(state: any): void;
   public abstract unstable_loadExternalState(state: any): void;
 
-  public get messages() {
+  protected _voiceMessages: ThreadMessage[] = [];
+  protected _voiceGeneration = 0;
+  private _cachedMergedMessages: readonly ThreadMessage[] | null = null;
+  private _cachedVoiceGeneration = -1;
+  private _cachedMergedBase: readonly ThreadMessage[] | null = null;
+
+  protected _markVoiceMessagesDirty() {
+    this._voiceGeneration++;
+    this._cachedMergedMessages = null;
+  }
+
+  protected _getBaseMessages(): readonly ThreadMessage[] {
     return this.repository.getMessages();
   }
 
+  public get messages(): readonly ThreadMessage[] {
+    if (this._voiceMessages.length === 0) {
+      return this._getBaseMessages();
+    }
+    const base = this._getBaseMessages();
+    if (
+      this._cachedVoiceGeneration !== this._voiceGeneration ||
+      this._cachedMergedBase !== base
+    ) {
+      this._cachedMergedMessages = [...base, ...this._voiceMessages];
+      this._cachedVoiceGeneration = this._voiceGeneration;
+      this._cachedMergedBase = base;
+    }
+    return this._cachedMergedMessages!;
+  }
+
   public get state() {
     let mostRecentAssistantMessage;
     for (const message of this.messages) {
@@ -99,11 +134,28 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
     try {
       return this.repository.getMessage(messageId);
     } catch {
+      // Check voice messages
+      const baseMessages = this.repository.getMessages();
+      const voiceIdx = this._voiceMessages.findIndex((m) => m.id === messageId);
+      if (voiceIdx !== -1) {
+        const parentId =
+          voiceIdx > 0
+            ? this._voiceMessages[voiceIdx - 1]!.id
+            : (baseMessages.at(-1)?.id ?? null);
+        return {
+          parentId,
+          message: this._voiceMessages[voiceIdx]!,
+          index: baseMessages.length + voiceIdx,
+        };
+      }
       return undefined;
     }
   }
 
   public getBranches(messageId: string): string[] {
+    if (this._voiceMessages.some((m) => m.id === messageId)) {
+      return [];
+    }
     return this.repository.getBranches(messageId);
   }
 
@@ -188,6 +240,195 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
     this._notifySubscribers();
   }
 
+  private _voiceSession: RealtimeVoiceAdapter.Session | undefined;
+  private _voiceUnsubs: Array<() => void> = [];
+  public voice: VoiceSessionState | undefined;
+
+  private _voiceVolume = 0;
+  private _voiceVolumeSubscribers = new Set<() => void>();
+
+  public getVoiceVolume = () => this._voiceVolume;
+
+  public subscribeVoiceVolume = (callback: () => void): Unsubscribe => {
+    this._voiceVolumeSubscribers.add(callback);
+    return () => this._voiceVolumeSubscribers.delete(callback);
+  };
+
+  public connectVoice() {
+    const adapter = this.adapters?.voice;
+    if (!adapter) throw new Error("Voice adapter not configured");
+
+    this.disconnectVoice();
+
+    const session = adapter.connect({});
+    this._voiceSession = session;
+    const unsubs: Array<() => void> = [];
+
+    let currentMode: RealtimeVoiceAdapter.Mode = "listening";
+
+    this.voice = {
+      status: session.status,
+      isMuted: session.isMuted,
+      mode: currentMode,
+    };
+    this._voiceVolume = 0;
+    this._notifySubscribers();
+
+    unsubs.push(
+      session.onStatusChange((status) => {
+        if (status.type === "ended") {
+          this._finishVoiceAssistantMessage();
+          this._voiceSession = undefined;
+          this.voice = undefined;
+        } else {
+          this.voice = {
+            status,
+            isMuted: session.isMuted,
+            mode: currentMode,
+          };
+        }
+        this._notifySubscribers();
+      }),
+    );
+
+    unsubs.push(
+      session.onModeChange((mode) => {
+        currentMode = mode;
+        if (this.voice) {
+          this.voice = { ...this.voice, mode };
+          this._notifySubscribers();
+        }
+      }),
+    );
+
+    unsubs.push(
+      session.onVolumeChange((volume) => {
+        this._voiceVolume = volume;
+        for (const cb of this._voiceVolumeSubscribers) cb();
+      }),
+    );
+
+    unsubs.push(
+      session.onTranscript((transcript) => {
+        this._handleVoiceTranscript(transcript);
+      }),
+    );
+
+    this._voiceUnsubs = unsubs;
+  }
+
+  private _currentAssistantMsg: ThreadAssistantMessage | null = null;
+
+  private _handleVoiceTranscript(
+    transcript: RealtimeVoiceAdapter.TranscriptItem,
+  ) {
+    this.ensureInitialized();
+
+    if (transcript.role === "user") {
+      this._finishVoiceAssistantMessage();
+      this._currentAssistantMsg = null;
+
+      if (transcript.isFinal) {
+        this._voiceMessages.push({
+          id: generateId(),
+          role: "user",
+          content: [{ type: "text", text: transcript.text }],
+          metadata: { custom: {} },
+          createdAt: new Date(),
+          status: { type: "complete", reason: "unknown" },
+          attachments: [],
+        });
+        this._markVoiceMessagesDirty();
+        this._notifySubscribers();
+      }
+    } else {
+      if (!this._currentAssistantMsg) {
+        this._currentAssistantMsg = {
+          id: generateId(),
+          role: "assistant",
+          content: [{ type: "text", text: transcript.text }],
+          metadata: {
+            unstable_state: this.state,
+            unstable_annotations: [],
+            unstable_data: [],
+            steps: [],
+            custom: {},
+          },
+          status: { type: "running" },
+          createdAt: new Date(),
+        };
+        this._voiceMessages.push(this._currentAssistantMsg);
+      } else {
+        const idx = this._voiceMessages.indexOf(this._currentAssistantMsg);
+        if (idx === -1) return;
+        const updated: ThreadAssistantMessage = {
+          ...this._currentAssistantMsg,
+          content: [{ type: "text", text: transcript.text }],
+          ...(transcript.isFinal
+            ? { status: { type: "complete", reason: "stop" } }
+            : {}),
+        };
+        this._voiceMessages[idx] = updated;
+        this._currentAssistantMsg = updated;
+      }
+
+      if (transcript.isFinal) {
+        this._currentAssistantMsg = null;
+      }
+
+      this._markVoiceMessagesDirty();
+      this._notifySubscribers();
+    }
+  }
+
+  private _finishVoiceAssistantMessage() {
+    const last = this._voiceMessages.at(-1);
+    if (last?.role === "assistant" && last.status.type === "running") {
+      const idx = this._voiceMessages.length - 1;
+      this._voiceMessages[idx] = {
+        ...(last as ThreadAssistantMessage),
+        status: { type: "complete", reason: "stop" },
+      };
+      this._markVoiceMessagesDirty();
+      this._notifySubscribers();
+    }
+  }
+
+  public disconnectVoice() {
+    this._finishVoiceAssistantMessage();
+    this._currentAssistantMsg = null;
+    for (const unsub of this._voiceUnsubs) unsub();
+    this._voiceUnsubs = [];
+    this._voiceSession?.disconnect();
+    this._voiceSession = undefined;
+    this.voice = undefined;
+    this._voiceVolume = 0;
+    for (const cb of this._voiceVolumeSubscribers) cb();
+    this._voiceMessages = [];
+    this._markVoiceMessagesDirty();
+    this._notifySubscribers();
+  }
+
+  public muteVoice() {
+    if (!this._voiceSession) throw new Error("No active voice session");
+    this._voiceSession.mute();
+    this.voice = {
+      ...this.voice!,
+      isMuted: true,
+    };
+    this._notifySubscribers();
+  }
+
+  public unmuteVoice() {
+    if (!this._voiceSession) throw new Error("No active voice session");
+    this._voiceSession.unmute();
+    this.voice = {
+      ...this.voice!,
+      isMuted: false,
+    };
+    this._notifySubscribers();
+  }
+
   protected ensureInitialized() {
     if (!this._isInitialized) {
       this._isInitialized = true;

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -4,6 +4,7 @@ import type { Unsubscribe } from "../../types/unsubscribe";
 import type { AppendMessage, ThreadMessage } from "../../types/message";
 import type { RunConfig } from "../../types/message";
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type {
   ChatModelRunOptions,
   ChatModelRunResult,
@@ -24,6 +25,7 @@ export type RuntimeCapabilities = {
   readonly unstable_copy: boolean;
   readonly speech: boolean;
   readonly dictation: boolean;
+  readonly voice: boolean;
   readonly attachments: boolean;
   readonly feedback: boolean;
   readonly queue: boolean;
@@ -57,6 +59,12 @@ export type SpeechState = {
   readonly status: SpeechSynthesisAdapter.Status;
 };
 
+export type VoiceSessionState = {
+  readonly status: RealtimeVoiceAdapter.Status;
+  readonly isMuted: boolean;
+  readonly mode: RealtimeVoiceAdapter.Mode;
+};
+
 export type SubmittedFeedback = {
   readonly type: "negative" | "positive";
 };
@@ -102,6 +110,11 @@ export type ThreadRuntimeCore = Readonly<{
   speak: (messageId: string) => void;
   stopSpeaking: () => void;
 
+  connectVoice: () => void;
+  disconnectVoice: () => void;
+  muteVoice: () => void;
+  unmuteVoice: () => void;
+
   submitFeedback: (feedback: SubmitFeedbackOptions) => void;
 
   getModelContext: () => ModelContext;
@@ -111,6 +124,7 @@ export type ThreadRuntimeCore = Readonly<{
   beginEdit: (messageId: string) => void;
 
   speech: SpeechState | undefined;
+  voice: VoiceSessionState | undefined;
 
   capabilities: Readonly<RuntimeCapabilities>;
   isDisabled: boolean;
@@ -123,6 +137,9 @@ export type ThreadRuntimeCore = Readonly<{
 
   subscribe: (callback: () => void) => Unsubscribe;
 
+  getVoiceVolume: () => number;
+  subscribeVoiceVolume: (callback: () => void) => Unsubscribe;
+
   import(repository: ExportedMessageRepository): void;
   export(): ExportedMessageRepository;
 

package/src/runtimes/external-store/external-store-adapter.ts

@@ -5,6 +5,7 @@ import type {
   SpeechSynthesisAdapter,
   DictationAdapter,
 } from "../../adapters/speech";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { FeedbackAdapter } from "../../adapters/feedback";
 import type {
   AddToolResultOptions,
@@ -90,6 +91,7 @@ type ExternalStoreAdapterBase<T> = {
         attachments?: AttachmentAdapter | undefined;
         speech?: SpeechSynthesisAdapter | undefined;
         dictation?: DictationAdapter | undefined;
+        voice?: RealtimeVoiceAdapter | undefined;
         feedback?: FeedbackAdapter | undefined;
         /**
          * @deprecated This API is still under active development and might change without notice.

package/src/runtimes/external-store/external-store-thread-runtime-core.ts

@@ -63,6 +63,7 @@ export class ExternalStoreThreadRuntimeCore
     unstable_copy: false,
     speech: false,
     dictation: false,
+    voice: false,
     attachments: false,
     feedback: false,
     queue: false,
@@ -78,7 +79,7 @@ export class ExternalStoreThreadRuntimeCore
     return this._store.isLoading ?? false;
   }
 
-  public override get messages() {
+  protected override _getBaseMessages(): readonly ThreadMessage[] {
     return this._messages;
   }
 
@@ -137,6 +138,7 @@ export class ExternalStoreThreadRuntimeCore
       cancel: this._store.onCancel !== undefined,
       speech: this._store.adapters?.speech !== undefined,
       dictation: this._store.adapters?.dictation !== undefined,
+      voice: this._store.adapters?.voice !== undefined,
       unstable_copy: this._store.unstable_capabilities?.copy !== false,
       attachments: !!this._store.adapters?.attachments,
       feedback: !!this._store.adapters?.feedback,

package/src/runtimes/local/local-runtime-options.ts

@@ -5,6 +5,7 @@ import type {
   SpeechSynthesisAdapter,
   DictationAdapter,
 } from "../../adapters/speech";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { SuggestionAdapter } from "../../adapters/suggestion";
 import type { ChatModelAdapter } from "../../runtime/utils/chat-model-adapter";
 
@@ -16,6 +17,7 @@ export type LocalRuntimeOptionsBase = {
     attachments?: AttachmentAdapter | undefined;
     speech?: SpeechSynthesisAdapter | undefined;
     dictation?: DictationAdapter | undefined;
+    voice?: RealtimeVoiceAdapter | undefined;
     feedback?: FeedbackAdapter | undefined;
     suggestion?: SuggestionAdapter | undefined;
   };

package/src/runtimes/local/local-thread-runtime-core.ts

@@ -45,6 +45,7 @@ export class LocalThreadRuntimeCore
     unstable_copy: true,
     speech: false,
     dictation: false,
+    voice: false,
     attachments: false,
     feedback: false,
     queue: false,
@@ -118,6 +119,12 @@ export class LocalThreadRuntimeCore
       hasUpdates = true;
     }
 
+    const canVoice = options.adapters?.voice !== undefined;
+    if (this.capabilities.voice !== canVoice) {
+      this.capabilities.voice = canVoice;
+      hasUpdates = true;
+    }
+
     const canAttach = options.adapters?.attachments !== undefined;
     if (this.capabilities.attachments !== canAttach) {
       this.capabilities.attachments = canAttach;