@assistant-ui/core 0.1.9 → 0.1.11

package/src/react/model-context/useToolArgsStatus.ts

@@ -0,0 +1,51 @@
+import { useMemo } from "react";
+import { useAuiState } from "@assistant-ui/store";
+import {
+  getPartialJsonObjectFieldState,
+  getPartialJsonObjectMeta,
+} from "assistant-stream/utils";
+
+type PropFieldStatus = "streaming" | "complete";
+
+export type ToolArgsStatus<
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  status: "running" | "complete" | "incomplete" | "requires-action";
+  propStatus: Partial<Record<keyof TArgs, PropFieldStatus>>;
+};
+
+export const useToolArgsStatus = <
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+>(): ToolArgsStatus<TArgs> => {
+  const part = useAuiState((s) => s.part);
+
+  return useMemo(() => {
+    const statusType = part.status.type;
+
+    if (part.type !== "tool-call") {
+      throw new Error(
+        "useToolArgsStatus can only be used inside tool-call message parts",
+      );
+    }
+
+    const isStreaming = statusType === "running";
+    const args = part.args as Record<string, unknown>;
+    const meta = getPartialJsonObjectMeta(args as Record<symbol, unknown>);
+    const propStatus: Partial<Record<string, PropFieldStatus>> = {};
+
+    for (const key of Object.keys(args)) {
+      if (meta) {
+        const fieldState = getPartialJsonObjectFieldState(args, [key]);
+        propStatus[key] =
+          fieldState === "complete" || !isStreaming ? "complete" : "streaming";
+      } else {
+        propStatus[key] = isStreaming ? "streaming" : "complete";
+      }
+    }
+
+    return {
+      status: statusType,
+      propStatus: propStatus as Partial<Record<keyof TArgs, PropFieldStatus>>,
+    };
+  }, [part]);
+};

package/src/react/primitive-hooks/useVoice.ts

@@ -0,0 +1,41 @@
+import { useCallback, useSyncExternalStore } from "react";
+import { useAui, useAuiState } from "@assistant-ui/store";
+import type { VoiceSessionState } from "../../runtime/interfaces/thread-runtime-core";
+
+export const useVoiceState = (): VoiceSessionState | undefined => {
+  return useAuiState((s) => s.thread.voice);
+};
+
+const getServerVolume = () => 0;
+
+export const useVoiceVolume = (): number => {
+  const aui = useAui();
+  const thread = aui.thread();
+  return useSyncExternalStore(
+    thread.subscribeVoiceVolume,
+    thread.getVoiceVolume,
+    getServerVolume,
+  );
+};
+
+export const useVoiceControls = () => {
+  const aui = useAui();
+
+  const connect = useCallback(() => {
+    aui.thread().connectVoice();
+  }, [aui]);
+
+  const disconnect = useCallback(() => {
+    aui.thread().disconnectVoice();
+  }, [aui]);
+
+  const mute = useCallback(() => {
+    aui.thread().muteVoice();
+  }, [aui]);
+
+  const unmute = useCallback(() => {
+    aui.thread().unmuteVoice();
+  }, [aui]);
+
+  return { connect, disconnect, mute, unmute };
+};

package/src/react/runtimes/RemoteThreadListThreadListRuntimeCore.tsx

@@ -39,7 +39,7 @@ export class RemoteThreadListThreadListRuntimeCore
 
   private _mainThreadId!: string;
   private readonly _state = new OptimisticState<RemoteThreadState>({
-    isLoading: false,
+    isLoading: true,
     newThreadId: undefined,
     threadIds: [],
     archivedThreadIds: [],
@@ -103,6 +103,7 @@ export class RemoteThreadListThreadListRuntimeCore
 
             return {
               ...state,
+              isLoading: false,
               threadIds: newThreadIds,
               archivedThreadIds: newArchivedThreadIds,
               threadIdMap: {
@@ -116,6 +117,13 @@ export class RemoteThreadListThreadListRuntimeCore
             };
           },
         })
+        .catch(() => {
+          this._loadThreadsPromise = undefined;
+          this._state.update({
+            ...this._state.baseValue,
+            isLoading: false,
+          });
+        })
         .then(() => {});
     }
 
@@ -141,7 +149,12 @@ export class RemoteThreadListThreadListRuntimeCore
         Fragment) as ComponentType<PropsWithChildren>,
     }));
     this.__internal_setOptions(options);
-    this.switchToNewThread();
+    const startThreadId = options.threadId ?? options.initialThreadId;
+    if (startThreadId) {
+      this.switchToThread(startThreadId);
+    } else {
+      this.switchToNewThread();
+    }
   }
 
   private useProvider;

package/src/react/runtimes/useRemoteThreadListRuntime.ts

@@ -34,6 +34,7 @@ const useRemoteThreadListRuntimeImpl = (
     runtime.threads.__internal_setOptions(options);
     runtime.threads.__internal_load();
   }, [runtime, options]);
+
   return useMemo(() => new AssistantRuntimeImpl(runtime), [runtime]);
 };
 
@@ -43,6 +44,9 @@ export const useRemoteThreadListRuntime = (
   const runtimeHookRef = useRef(options.runtimeHook);
   runtimeHookRef.current = options.runtimeHook;
 
+  // threadId/initialThreadId only affect the constructor; capture once via ref
+  const startThreadIdRef = useRef(options.threadId ?? options.initialThreadId);
+
   const stableRuntimeHook = useCallback(() => {
     return runtimeHookRef.current();
   }, []);
@@ -51,6 +55,7 @@ export const useRemoteThreadListRuntime = (
     () => ({
       adapter: options.adapter,
       allowNesting: options.allowNesting,
+      initialThreadId: startThreadIdRef.current,
       runtimeHook: stableRuntimeHook,
     }),
     [options.adapter, options.allowNesting, stableRuntimeHook],
@@ -72,5 +77,18 @@ export const useRemoteThreadListRuntime = (
     return stableRuntimeHook();
   }
 
-  return useRemoteThreadListRuntimeImpl(stableOptions);
+  const runtime = useRemoteThreadListRuntimeImpl(stableOptions);
+
+  const prevThreadIdRef = useRef(options.threadId);
+  useEffect(() => {
+    if (options.threadId === prevThreadIdRef.current) return;
+    prevThreadIdRef.current = options.threadId;
+    if (options.threadId) {
+      runtime.threads.switchToThread(options.threadId).catch(() => {});
+    } else {
+      runtime.threads.switchToNewThread().catch(() => {});
+    }
+  }, [runtime, options.threadId]);
+
+  return runtime;
 };

package/src/react/runtimes/useToolInvocations.ts

@@ -360,6 +360,13 @@ export function useToolInvocations({
               }
               let lastState = lastToolStates.current[content.toolCallId];
               if (!lastState) {
+                if (content.result !== undefined) {
+                  if (content.messages) {
+                    processMessages(content.messages);
+                  }
+                  return;
+                }
+
                 toolCallIdAliasesRef.current.set(
                   content.toolCallId,
                   content.toolCallId,
@@ -567,6 +574,8 @@ export function useToolInvocations({
   return {
     reset: () => {
       isInitialState.current = true;
+      ignoredToolIds.current.clear();
+      lastToolStates.current = {};
       void abort().finally(() => {
         startedExecutionToolCallIdsRef.current.clear();
         toolCallIdAliasesRef.current.clear();

package/src/react/types/scopes/interactables.ts

@@ -27,9 +27,25 @@ export type InteractableRegistration = {
   selected?: boolean | undefined;
 };
 
+export type InteractablePersistenceStatus = {
+  isPending: boolean;
+  error: unknown;
+};
+
 export type InteractablesState = {
   /** Keyed by instance id */
   definitions: Record<string, InteractableDefinition>;
+  /** Per-id persistence sync status */
+  persistence: Record<string, InteractablePersistenceStatus>;
+};
+
+export type InteractablePersistedState = Record<
+  string,
+  { name: string; state: unknown }
+>;
+
+export type InteractablePersistenceAdapter = {
+  save(state: InteractablePersistedState): void | Promise<void>;
 };
 
 export type InteractablesMethods = {
@@ -37,6 +53,12 @@ export type InteractablesMethods = {
   register(def: InteractableRegistration): Unsubscribe;
   setState(id: string, updater: (prev: unknown) => unknown): void;
   setSelected(id: string, selected: boolean): void;
+  exportState(): InteractablePersistedState;
+  importState(saved: InteractablePersistedState): void;
+  setPersistenceAdapter(
+    adapter: InteractablePersistenceAdapter | undefined,
+  ): void;
+  flush(): Promise<void>;
 };
 
 export type InteractablesClientSchema = {

package/src/runtime/api/thread-list-runtime.ts

@@ -46,6 +46,8 @@ export type ThreadListRuntime = {
 
   switchToThread(threadId: string): Promise<void>;
   switchToNewThread(): Promise<void>;
+
+  getLoadThreadsPromise(): Promise<void>;
 };
 
 const getThreadListState = (
@@ -130,6 +132,7 @@ export class ThreadListRuntimeImpl implements ThreadListRuntime {
   protected __internal_bindMethods() {
     this.switchToThread = this.switchToThread.bind(this);
     this.switchToNewThread = this.switchToNewThread.bind(this);
+    this.getLoadThreadsPromise = this.getLoadThreadsPromise.bind(this);
     this.getState = this.getState.bind(this);
     this.subscribe = this.subscribe.bind(this);
     this.getById = this.getById.bind(this);
@@ -146,6 +149,10 @@ export class ThreadListRuntimeImpl implements ThreadListRuntime {
     return this._core.switchToNewThread();
   }
 
+  public getLoadThreadsPromise(): Promise<void> {
+    return this._core.getLoadThreadsPromise();
+  }
+
   public getState(): ThreadListState {
     return this._getState();
   }

package/src/runtime/api/thread-runtime.ts

@@ -3,6 +3,7 @@ import {
   RuntimeCapabilities,
   ThreadRuntimeCore,
   SpeechState,
+  VoiceSessionState,
   ThreadRuntimeEventType,
   StartRunConfig,
   ResumeRunConfig,
@@ -187,6 +188,8 @@ export type ThreadState = {
    * @deprecated This API is still under active development and might change without notice.
    */
   readonly speech: SpeechState | undefined;
+
+  readonly voice: VoiceSessionState | undefined;
 };
 
 export const getThreadState = (
@@ -209,6 +212,7 @@ export const getThreadState = (
     suggestions: runtime.suggestions,
     extras: runtime.extras,
     speech: runtime.speech,
+    voice: runtime.voice,
   });
 };
 
@@ -317,6 +321,13 @@ export type ThreadRuntime = {
    */
   stopSpeaking(): void;
 
+  connectVoice(): void;
+  disconnectVoice(): void;
+  getVoiceVolume(): number;
+  subscribeVoiceVolume(callback: () => void): Unsubscribe;
+  muteVoice(): void;
+  unmuteVoice(): void;
+
   unstable_on(event: ThreadRuntimeEventType, callback: () => void): Unsubscribe;
 };
 
@@ -388,6 +399,12 @@ export class ThreadRuntimeImpl implements ThreadRuntime {
     this.startRun = this.startRun.bind(this);
     this.cancelRun = this.cancelRun.bind(this);
     this.stopSpeaking = this.stopSpeaking.bind(this);
+    this.connectVoice = this.connectVoice.bind(this);
+    this.disconnectVoice = this.disconnectVoice.bind(this);
+    this.muteVoice = this.muteVoice.bind(this);
+    this.unmuteVoice = this.unmuteVoice.bind(this);
+    this.getVoiceVolume = this.getVoiceVolume.bind(this);
+    this.subscribeVoiceVolume = this.subscribeVoiceVolume.bind(this);
     this.export = this.export.bind(this);
     this.import = this.import.bind(this);
     this.reset = this.reset.bind(this);
@@ -463,6 +480,30 @@ export class ThreadRuntimeImpl implements ThreadRuntime {
     return this._threadBinding.getState().stopSpeaking();
   }
 
+  public connectVoice() {
+    this._threadBinding.getState().connectVoice();
+  }
+
+  public disconnectVoice() {
+    this._threadBinding.getState().disconnectVoice();
+  }
+
+  public getVoiceVolume() {
+    return this._threadBinding.getState().getVoiceVolume();
+  }
+
+  public subscribeVoiceVolume(callback: () => void) {
+    return this._threadBinding.getState().subscribeVoiceVolume(callback);
+  }
+
+  public muteVoice() {
+    this._threadBinding.getState().muteVoice();
+  }
+
+  public unmuteVoice() {
+    this._threadBinding.getState().unmuteVoice();
+  }
+
   public export() {
     return this._threadBinding.getState().export();
   }

package/src/runtime/base/base-thread-runtime-core.ts

@@ -1,7 +1,12 @@
-import type { AppendMessage, ThreadMessage } from "../../types/message";
+import type {
+  AppendMessage,
+  ThreadAssistantMessage,
+  ThreadMessage,
+} from "../../types/message";
 import type { Unsubscribe } from "../../types/unsubscribe";
 import type { ModelContextProvider } from "../../model-context/types";
 import { getThreadMessageText } from "../../utils/text";
+import { generateId } from "../../utils/id";
 import {
   ExportedMessageRepository,
   MessageRepository,
@@ -14,6 +19,7 @@ import type {
   SubmitFeedbackOptions,
   ThreadRuntimeCore,
   SpeechState,
+  VoiceSessionState,
   RuntimeCapabilities,
   ThreadRuntimeEventType,
   StartRunConfig,
@@ -23,12 +29,14 @@ import { DefaultEditComposerRuntimeCore } from "./default-edit-composer-runtime-
 import type { SpeechSynthesisAdapter } from "../../adapters/speech";
 import type { FeedbackAdapter } from "../../adapters/feedback";
 import type { AttachmentAdapter } from "../../adapters/attachment";
+import type { RealtimeVoiceAdapter } from "../../adapters/voice";
 import type { ThreadMessageLike } from "../utils/thread-message-like";
 
 type BaseThreadAdapters = {
   speech?: SpeechSynthesisAdapter | undefined;
   feedback?: FeedbackAdapter | undefined;
   attachments?: AttachmentAdapter | undefined;
+  voice?: RealtimeVoiceAdapter | undefined;
 };
 
 export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
@@ -53,10 +61,37 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
   public abstract importExternalState(state: any): void;
   public abstract unstable_loadExternalState(state: any): void;
 
-  public get messages() {
+  protected _voiceMessages: ThreadMessage[] = [];
+  protected _voiceGeneration = 0;
+  private _cachedMergedMessages: readonly ThreadMessage[] | null = null;
+  private _cachedVoiceGeneration = -1;
+  private _cachedMergedBase: readonly ThreadMessage[] | null = null;
+
+  protected _markVoiceMessagesDirty() {
+    this._voiceGeneration++;
+    this._cachedMergedMessages = null;
+  }
+
+  protected _getBaseMessages(): readonly ThreadMessage[] {
     return this.repository.getMessages();
   }
 
+  public get messages(): readonly ThreadMessage[] {
+    if (this._voiceMessages.length === 0) {
+      return this._getBaseMessages();
+    }
+    const base = this._getBaseMessages();
+    if (
+      this._cachedVoiceGeneration !== this._voiceGeneration ||
+      this._cachedMergedBase !== base
+    ) {
+      this._cachedMergedMessages = [...base, ...this._voiceMessages];
+      this._cachedVoiceGeneration = this._voiceGeneration;
+      this._cachedMergedBase = base;
+    }
+    return this._cachedMergedMessages!;
+  }
+
   public get state() {
     let mostRecentAssistantMessage;
     for (const message of this.messages) {
@@ -99,11 +134,28 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
     try {
       return this.repository.getMessage(messageId);
     } catch {
+      // Check voice messages
+      const baseMessages = this.repository.getMessages();
+      const voiceIdx = this._voiceMessages.findIndex((m) => m.id === messageId);
+      if (voiceIdx !== -1) {
+        const parentId =
+          voiceIdx > 0
+            ? this._voiceMessages[voiceIdx - 1]!.id
+            : (baseMessages.at(-1)?.id ?? null);
+        return {
+          parentId,
+          message: this._voiceMessages[voiceIdx]!,
+          index: baseMessages.length + voiceIdx,
+        };
+      }
       return undefined;
     }
   }
 
   public getBranches(messageId: string): string[] {
+    if (this._voiceMessages.some((m) => m.id === messageId)) {
+      return [];
+    }
     return this.repository.getBranches(messageId);
   }
 
@@ -188,6 +240,195 @@ export abstract class BaseThreadRuntimeCore implements ThreadRuntimeCore {
     this._notifySubscribers();
   }
 
+  private _voiceSession: RealtimeVoiceAdapter.Session | undefined;
+  private _voiceUnsubs: Array<() => void> = [];
+  public voice: VoiceSessionState | undefined;
+
+  private _voiceVolume = 0;
+  private _voiceVolumeSubscribers = new Set<() => void>();
+
+  public getVoiceVolume = () => this._voiceVolume;
+
+  public subscribeVoiceVolume = (callback: () => void): Unsubscribe => {
+    this._voiceVolumeSubscribers.add(callback);
+    return () => this._voiceVolumeSubscribers.delete(callback);
+  };
+
+  public connectVoice() {
+    const adapter = this.adapters?.voice;
+    if (!adapter) throw new Error("Voice adapter not configured");
+
+    this.disconnectVoice();
+
+    const session = adapter.connect({});
+    this._voiceSession = session;
+    const unsubs: Array<() => void> = [];
+
+    let currentMode: RealtimeVoiceAdapter.Mode = "listening";
+
+    this.voice = {
+      status: session.status,
+      isMuted: session.isMuted,
+      mode: currentMode,
+    };
+    this._voiceVolume = 0;
+    this._notifySubscribers();
+
+    unsubs.push(
+      session.onStatusChange((status) => {
+        if (status.type === "ended") {
+          this._finishVoiceAssistantMessage();
+          this._voiceSession = undefined;
+          this.voice = undefined;
+        } else {
+          this.voice = {
+            status,
+            isMuted: session.isMuted,
+            mode: currentMode,
+          };
+        }
+        this._notifySubscribers();
+      }),
+    );
+
+    unsubs.push(
+      session.onModeChange((mode) => {
+        currentMode = mode;
+        if (this.voice) {
+          this.voice = { ...this.voice, mode };
+          this._notifySubscribers();
+        }
+      }),
+    );
+
+    unsubs.push(
+      session.onVolumeChange((volume) => {
+        this._voiceVolume = volume;
+        for (const cb of this._voiceVolumeSubscribers) cb();
+      }),
+    );
+
+    unsubs.push(
+      session.onTranscript((transcript) => {
+        this._handleVoiceTranscript(transcript);
+      }),
+    );
+
+    this._voiceUnsubs = unsubs;
+  }
+
+  private _currentAssistantMsg: ThreadAssistantMessage | null = null;
+
+  private _handleVoiceTranscript(
+    transcript: RealtimeVoiceAdapter.TranscriptItem,
+  ) {
+    this.ensureInitialized();
+
+    if (transcript.role === "user") {
+      this._finishVoiceAssistantMessage();
+      this._currentAssistantMsg = null;
+
+      if (transcript.isFinal) {
+        this._voiceMessages.push({
+          id: generateId(),
+          role: "user",
+          content: [{ type: "text", text: transcript.text }],
+          metadata: { custom: {} },
+          createdAt: new Date(),
+          status: { type: "complete", reason: "unknown" },
+          attachments: [],
+        });
+        this._markVoiceMessagesDirty();
+        this._notifySubscribers();
+      }
+    } else {
+      if (!this._currentAssistantMsg) {
+        this._currentAssistantMsg = {
+          id: generateId(),
+          role: "assistant",
+          content: [{ type: "text", text: transcript.text }],
+          metadata: {
+            unstable_state: this.state,
+            unstable_annotations: [],
+            unstable_data: [],
+            steps: [],
+            custom: {},
+          },
+          status: { type: "running" },
+          createdAt: new Date(),
+        };
+        this._voiceMessages.push(this._currentAssistantMsg);
+      } else {
+        const idx = this._voiceMessages.indexOf(this._currentAssistantMsg);
+        if (idx === -1) return;
+        const updated: ThreadAssistantMessage = {
+          ...this._currentAssistantMsg,
+          content: [{ type: "text", text: transcript.text }],
+          ...(transcript.isFinal
+            ? { status: { type: "complete", reason: "stop" } }
+            : {}),
+        };
+        this._voiceMessages[idx] = updated;
+        this._currentAssistantMsg = updated;
+      }
+
+      if (transcript.isFinal) {
+        this._currentAssistantMsg = null;
+      }
+
+      this._markVoiceMessagesDirty();
+      this._notifySubscribers();
+    }
+  }
+
+  private _finishVoiceAssistantMessage() {
+    const last = this._voiceMessages.at(-1);
+    if (last?.role === "assistant" && last.status.type === "running") {
+      const idx = this._voiceMessages.length - 1;
+      this._voiceMessages[idx] = {
+        ...(last as ThreadAssistantMessage),
+        status: { type: "complete", reason: "stop" },
+      };
+      this._markVoiceMessagesDirty();
+      this._notifySubscribers();
+    }
+  }
+
+  public disconnectVoice() {
+    this._finishVoiceAssistantMessage();
+    this._currentAssistantMsg = null;
+    for (const unsub of this._voiceUnsubs) unsub();
+    this._voiceUnsubs = [];
+    this._voiceSession?.disconnect();
+    this._voiceSession = undefined;
+    this.voice = undefined;
+    this._voiceVolume = 0;
+    for (const cb of this._voiceVolumeSubscribers) cb();
+    this._voiceMessages = [];
+    this._markVoiceMessagesDirty();
+    this._notifySubscribers();
+  }
+
+  public muteVoice() {
+    if (!this._voiceSession) throw new Error("No active voice session");
+    this._voiceSession.mute();
+    this.voice = {
+      ...this.voice!,
+      isMuted: true,
+    };
+    this._notifySubscribers();
+  }
+
+  public unmuteVoice() {
+    if (!this._voiceSession) throw new Error("No active voice session");
+    this._voiceSession.unmute();
+    this.voice = {
+      ...this.voice!,
+      isMuted: false,
+    };
+    this._notifySubscribers();
+  }
+
   protected ensureInitialized() {
     if (!this._isInitialized) {
       this._isInitialized = true;