@assistant-ui/core 0.1.8 → 0.1.10

package/src/adapters/voice.ts

@@ -0,0 +1,166 @@
+import type { Unsubscribe } from "../types/unsubscribe";
+
+export namespace RealtimeVoiceAdapter {
+  export type Status =
+    | {
+        type: "starting" | "running";
+      }
+    | {
+        type: "ended";
+        reason: "finished" | "cancelled" | "error";
+        error?: unknown;
+      };
+
+  export type Mode = "listening" | "speaking";
+
+  export type TranscriptItem = {
+    role: "user" | "assistant";
+    text: string;
+    isFinal?: boolean;
+  };
+
+  export type Session = {
+    status: Status;
+    isMuted: boolean;
+
+    disconnect: () => void;
+    mute: () => void;
+    unmute: () => void;
+
+    onStatusChange: (callback: (status: Status) => void) => Unsubscribe;
+    onTranscript: (
+      callback: (transcript: TranscriptItem) => void,
+    ) => Unsubscribe;
+    onModeChange: (callback: (mode: Mode) => void) => Unsubscribe;
+    onVolumeChange: (callback: (volume: number) => void) => Unsubscribe;
+  };
+}
+
+export type RealtimeVoiceAdapter = {
+  connect: (options: {
+    abortSignal?: AbortSignal;
+  }) => RealtimeVoiceAdapter.Session;
+};
+
+export type VoiceSessionControls = {
+  disconnect: () => void;
+  mute: () => void;
+  unmute: () => void;
+};
+
+export type VoiceSessionHelpers = {
+  setStatus: (status: RealtimeVoiceAdapter.Status) => void;
+  end: (reason: "finished" | "cancelled" | "error", error?: unknown) => void;
+  emitTranscript: (item: RealtimeVoiceAdapter.TranscriptItem) => void;
+  emitMode: (mode: RealtimeVoiceAdapter.Mode) => void;
+  emitVolume: (volume: number) => void;
+  isDisposed: () => boolean;
+};
+
+export function createVoiceSession(
+  options: { abortSignal?: AbortSignal },
+  setup: (helpers: VoiceSessionHelpers) => Promise<VoiceSessionControls>,
+): RealtimeVoiceAdapter.Session {
+  const statusCbs = new Set<(s: RealtimeVoiceAdapter.Status) => void>();
+  const transcriptCbs = new Set<
+    (t: RealtimeVoiceAdapter.TranscriptItem) => void
+  >();
+  const modeCbs = new Set<(m: RealtimeVoiceAdapter.Mode) => void>();
+  const volumeCbs = new Set<(v: number) => void>();
+
+  let currentStatus: RealtimeVoiceAdapter.Status = { type: "starting" };
+  let isMuted = false;
+  let disposed = false;
+  let controls: VoiceSessionControls | null = null;
+
+  const cleanup = () => {
+    disposed = true;
+    statusCbs.clear();
+    transcriptCbs.clear();
+    modeCbs.clear();
+    volumeCbs.clear();
+  };
+
+  const helpers: VoiceSessionHelpers = {
+    setStatus: (status) => {
+      if (disposed) return;
+      currentStatus = status;
+      for (const cb of statusCbs) cb(status);
+    },
+    end: (reason, error?) => {
+      if (disposed) return;
+      currentStatus = { type: "ended", reason, error };
+      for (const cb of statusCbs) cb(currentStatus);
+      cleanup();
+    },
+    emitTranscript: (item) => {
+      if (disposed) return;
+      for (const cb of transcriptCbs) cb(item);
+    },
+    emitMode: (mode) => {
+      if (disposed) return;
+      for (const cb of modeCbs) cb(mode);
+    },
+    emitVolume: (volume) => {
+      if (disposed) return;
+      for (const cb of volumeCbs) cb(volume);
+    },
+    isDisposed: () => disposed,
+  };
+
+  const session: RealtimeVoiceAdapter.Session = {
+    get status() {
+      return currentStatus;
+    },
+    get isMuted() {
+      return isMuted;
+    },
+    disconnect: () => {
+      controls?.disconnect();
+      cleanup();
+    },
+    mute: () => {
+      controls?.mute();
+      isMuted = true;
+    },
+    unmute: () => {
+      controls?.unmute();
+      isMuted = false;
+    },
+    onStatusChange: (cb) => {
+      statusCbs.add(cb);
+      return () => statusCbs.delete(cb);
+    },
+    onTranscript: (cb) => {
+      transcriptCbs.add(cb);
+      return () => transcriptCbs.delete(cb);
+    },
+    onModeChange: (cb) => {
+      modeCbs.add(cb);
+      return () => modeCbs.delete(cb);
+    },
+    onVolumeChange: (cb) => {
+      volumeCbs.add(cb);
+      return () => volumeCbs.delete(cb);
+    },
+  };
+
+  if (options.abortSignal) {
+    options.abortSignal.addEventListener("abort", () => session.disconnect(), {
+      once: true,
+    });
+  }
+
+  const doSetup = async () => {
+    try {
+      if (disposed) return;
+      controls = await setup(helpers);
+      if (disposed) controls.disconnect();
+    } catch (error) {
+      helpers.end("error", error);
+    }
+  };
+
+  doSetup();
+  return session;
+}

package/src/index.ts

@@ -62,6 +62,7 @@ export type {
   // Tool & instruction config
   AssistantToolProps,
   AssistantInstructionsConfig,
+  AssistantContextConfig,
 } from "./model-context/types";
 export { mergeModelContexts } from "./model-context/types";
 
@@ -104,6 +105,14 @@ export {
   WebSpeechDictationAdapter,
 } from "./adapters/speech";
 
+// Voice adapter
+export type { RealtimeVoiceAdapter } from "./adapters/voice";
+export { createVoiceSession } from "./adapters/voice";
+export type {
+  VoiceSessionControls,
+  VoiceSessionHelpers,
+} from "./adapters/voice";
+
 // Feedback adapter
 export type { FeedbackAdapter } from "./adapters/feedback";
 
@@ -151,6 +160,7 @@ export type {
   SubmitFeedbackOptions,
   ThreadSuggestion,
   SpeechState,
+  VoiceSessionState,
   SubmittedFeedback,
   ThreadRuntimeEventType,
   StartRunConfig,

package/src/model-context/types.ts

@@ -55,6 +55,11 @@ export type AssistantInstructionsConfig = {
   instruction: string;
 };
 
+export type AssistantContextConfig = {
+  getContext: () => string;
+  disabled?: boolean | undefined;
+};
+
 // =============================================================================
 // Merging
 // =============================================================================

package/src/react/client/Interactables.ts

@@ -0,0 +1,332 @@
+import {
+  resource,
+  tapState,
+  tapEffect,
+  tapCallback,
+  tapRef,
+  tapMemo,
+} from "@assistant-ui/tap";
+import {
+  tapAssistantClientRef,
+  type ClientOutput,
+  attachTransformScopes,
+} from "@assistant-ui/store";
+import type {
+  InteractablesState,
+  InteractableRegistration,
+  InteractableStateSchema,
+  InteractablePersistedState,
+  InteractablePersistenceAdapter,
+} from "../types/scopes/interactables";
+import { toJSONSchema, toPartialJSONSchema } from "assistant-stream";
+import { ModelContext } from "../../store";
+import { buildInteractableModelContext } from "./interactable-model-context";
+
+const PERSISTENCE_DEBOUNCE_MS = 500;
+
+export const Interactables = resource((): ClientOutput<"interactables"> => {
+  const [state, setState] = tapState<InteractablesState>(() => ({
+    definitions: {},
+    persistence: {},
+  }));
+
+  const clientRef = tapAssistantClientRef();
+
+  const stateRef = tapRef(state);
+  tapEffect(() => {
+    stateRef.current = state;
+  }, [state]);
+
+  const subscribersRef = tapRef(new Set<() => void>());
+  const partialSchemaCacheRef = tapRef(
+    new Map<string, InteractableStateSchema>(),
+  );
+  const detachedStateRef = tapRef(new Map<string, unknown>());
+
+  const adapterRef = tapRef<InteractablePersistenceAdapter | undefined>(
+    undefined,
+  );
+  const debounceTimerRef = tapRef<ReturnType<typeof setTimeout> | undefined>(
+    undefined,
+  );
+  const syncSeqRef = tapRef(0);
+  const hasPendingLocalChangeRef = tapRef(false);
+  const flushResolversRef = tapRef<Array<() => void>>([]);
+  const dirtyIdsRef = tapRef(new Set<string>());
+
+  const runPersistence = tapCallback(async () => {
+    const adapter = adapterRef.current;
+    if (!adapter) {
+      for (const resolve of flushResolversRef.current) resolve();
+      flushResolversRef.current = [];
+      return;
+    }
+
+    const seq = ++syncSeqRef.current;
+    const dirtyIds = new Set(dirtyIdsRef.current);
+    dirtyIdsRef.current.clear();
+    hasPendingLocalChangeRef.current = true;
+
+    // Snapshot before any await so unregistered definitions are still included.
+    const exported = stateRef.current.definitions;
+    const payload: InteractablePersistedState = {};
+    for (const [id, def] of Object.entries(exported)) {
+      payload[id] = { name: def.name, state: def.state };
+    }
+
+    setState((prev) => ({
+      ...prev,
+      persistence: {
+        ...prev.persistence,
+        ...Object.fromEntries(
+          [...dirtyIds].map((id) => [
+            id,
+            { isPending: true, error: undefined },
+          ]),
+        ),
+      },
+    }));
+
+    try {
+      await adapter.save(payload);
+      if (syncSeqRef.current === seq) {
+        hasPendingLocalChangeRef.current = false;
+        setState((prev) => {
+          const persistence = { ...prev.persistence };
+          for (const id of dirtyIds) delete persistence[id];
+          return { ...prev, persistence };
+        });
+      }
+    } catch (e) {
+      if (syncSeqRef.current === seq) {
+        hasPendingLocalChangeRef.current = false;
+        setState((prev) => ({
+          ...prev,
+          persistence: {
+            ...prev.persistence,
+            ...Object.fromEntries(
+              [...dirtyIds].map((id) => [id, { isPending: false, error: e }]),
+            ),
+          },
+        }));
+      }
+    } finally {
+      if (dirtyIdsRef.current.size > 0 && adapterRef.current) {
+        runPersistence();
+      } else {
+        for (const resolve of flushResolversRef.current) resolve();
+        flushResolversRef.current = [];
+      }
+    }
+  }, []);
+
+  const schedulePersistence = tapCallback(
+    (id: string) => {
+      if (!adapterRef.current) return;
+      dirtyIdsRef.current.add(id);
+      if (debounceTimerRef.current !== undefined) {
+        clearTimeout(debounceTimerRef.current);
+      }
+      debounceTimerRef.current = setTimeout(() => {
+        debounceTimerRef.current = undefined;
+        if (!hasPendingLocalChangeRef.current) {
+          runPersistence();
+        } else {
+          debounceTimerRef.current = setTimeout(() => {
+            debounceTimerRef.current = undefined;
+            runPersistence();
+          }, PERSISTENCE_DEBOUNCE_MS);
+        }
+      }, PERSISTENCE_DEBOUNCE_MS);
+    },
+    [runPersistence],
+  );
+
+  const exportState = tapCallback((): InteractablePersistedState => {
+    const result: InteractablePersistedState = {};
+    for (const [id, def] of Object.entries(stateRef.current.definitions)) {
+      result[id] = { name: def.name, state: def.state };
+    }
+    return result;
+  }, []);
+
+  const importState = tapCallback((saved: InteractablePersistedState) => {
+    for (const [id, entry] of Object.entries(saved)) {
+      detachedStateRef.current.set(id, entry.state);
+    }
+    setState((prev) => {
+      let changed = false;
+      const definitions = { ...prev.definitions };
+      for (const [id, entry] of Object.entries(saved)) {
+        if (definitions[id]) {
+          definitions[id] = { ...definitions[id], state: entry.state };
+          changed = true;
+        }
+      }
+      return changed ? { ...prev, definitions } : prev;
+    });
+  }, []);
+
+  const setPersistenceAdapter = tapCallback(
+    (adapter: InteractablePersistenceAdapter | undefined) => {
+      adapterRef.current = adapter;
+    },
+    [],
+  );
+
+  const flush = tapCallback(async () => {
+    if (debounceTimerRef.current !== undefined) {
+      clearTimeout(debounceTimerRef.current);
+      debounceTimerRef.current = undefined;
+    }
+    if (!adapterRef.current) return;
+    if (!hasPendingLocalChangeRef.current && dirtyIdsRef.current.size === 0)
+      return;
+    const p = new Promise<void>((resolve) => {
+      flushResolversRef.current.push(resolve);
+    });
+    if (!hasPendingLocalChangeRef.current) {
+      runPersistence();
+    }
+    return p;
+  }, [runPersistence]);
+
+  const flushIfPending = tapCallback(() => {
+    if (adapterRef.current && debounceTimerRef.current !== undefined) {
+      clearTimeout(debounceTimerRef.current);
+      debounceTimerRef.current = undefined;
+      runPersistence();
+    }
+  }, [runPersistence]);
+
+  const setDefState = tapCallback(
+    (id: string, updater: (prev: unknown) => unknown) => {
+      setState((prev) => {
+        const existing = prev.definitions[id];
+        if (!existing) return prev;
+        return {
+          ...prev,
+          definitions: {
+            ...prev.definitions,
+            [id]: { ...existing, state: updater(existing.state) },
+          },
+        };
+      });
+      if (stateRef.current.definitions[id]) schedulePersistence(id);
+    },
+    [schedulePersistence],
+  );
+
+  const setDefSelected = tapCallback((id: string, selected: boolean) => {
+    setState((prev) => {
+      const existing = prev.definitions[id];
+      if (!existing) return prev;
+      return {
+        ...prev,
+        definitions: {
+          ...prev.definitions,
+          [id]: { ...existing, selected },
+        },
+      };
+    });
+  }, []);
+
+  const provider = tapMemo(
+    () => ({
+      getModelContext: () => {
+        const defs = stateRef.current.definitions;
+        return (
+          buildInteractableModelContext(
+            defs,
+            partialSchemaCacheRef.current,
+            setDefState,
+          ) ?? {}
+        );
+      },
+      subscribe: (callback: () => void) => {
+        subscribersRef.current.add(callback);
+        return () => {
+          subscribersRef.current.delete(callback);
+        };
+      },
+    }),
+    [setDefState],
+  );
+
+  // biome-ignore lint/correctness/useExhaustiveDependencies: state dep triggers notification
+  tapEffect(() => {
+    for (const cb of subscribersRef.current) cb();
+  }, [state]);
+
+  tapEffect(() => {
+    return clientRef.current!.modelContext().register(provider);
+  }, [clientRef, provider]);
+
+  const register = tapCallback(
+    (def: InteractableRegistration) => {
+      try {
+        const jsonSchema = toJSONSchema(def.stateSchema);
+        partialSchemaCacheRef.current.set(
+          def.id,
+          toPartialJSONSchema(jsonSchema),
+        );
+      } catch (e) {
+        console.warn(
+          `[Interactables] Failed to create partial schema for "${def.name}". The update tool will require all fields.`,
+          e,
+        );
+      }
+
+      const detached = detachedStateRef.current.get(def.id);
+      detachedStateRef.current.delete(def.id);
+
+      setState((prev) => ({
+        ...prev,
+        definitions: {
+          ...prev.definitions,
+          [def.id]: {
+            id: def.id,
+            name: def.name,
+            description: def.description,
+            stateSchema: def.stateSchema,
+            state:
+              prev.definitions[def.id]?.state ?? detached ?? def.initialState,
+            selected: def.selected,
+          },
+        },
+      }));
+
+      return () => {
+        flushIfPending();
+        setState((prev) => {
+          const existing = prev.definitions[def.id];
+          if (existing) {
+            detachedStateRef.current.set(def.id, existing.state);
+          }
+          partialSchemaCacheRef.current.delete(def.id);
+          const { [def.id]: _, ...rest } = prev.definitions;
+          const { [def.id]: __, ...restPersistence } = prev.persistence;
+          return { ...prev, definitions: rest, persistence: restPersistence };
+        });
+      };
+    },
+    [flushIfPending],
+  );
+
+  return {
+    getState: () => state,
+    register,
+    setState: setDefState,
+    setSelected: setDefSelected,
+    exportState,
+    importState,
+    setPersistenceAdapter,
+    flush,
+  };
+});
+
+attachTransformScopes(Interactables, (scopes, parent) => {
+  if (!scopes.modelContext && parent.modelContext.source === null) {
+    scopes.modelContext = ModelContext();
+  }
+});

package/src/react/client/interactable-model-context.ts

@@ -0,0 +1,83 @@
+import type { Tool } from "assistant-stream";
+import type {
+  InteractableDefinition,
+  InteractableStateSchema,
+} from "../types/scopes/interactables";
+
+export function shallowMerge(prev: unknown, partial: unknown): unknown {
+  if (
+    typeof prev !== "object" ||
+    prev === null ||
+    typeof partial !== "object" ||
+    partial === null ||
+    Array.isArray(prev) ||
+    Array.isArray(partial)
+  ) {
+    return partial;
+  }
+  return {
+    ...(prev as Record<string, unknown>),
+    ...(partial as Record<string, unknown>),
+  };
+}
+
+export function buildInteractableModelContext(
+  definitions: Record<string, InteractableDefinition>,
+  partialSchemaCache: Map<string, InteractableStateSchema>,
+  setDefState: (id: string, updater: (prev: unknown) => unknown) => void,
+): { system: string; tools: Record<string, Tool<any, any>> } | undefined {
+  const entries = Object.values(definitions);
+  if (entries.length === 0) return undefined;
+
+  const byName = new Map<string, InteractableDefinition[]>();
+  for (const def of entries) {
+    const list = byName.get(def.name) ?? [];
+    list.push(def);
+    byName.set(def.name, list);
+  }
+
+  const systemParts: string[] = [];
+  const tools: Record<string, Tool<any, any>> = {};
+
+  for (const [name, instances] of byName) {
+    const isMulti = instances.length > 1;
+
+    for (const def of instances) {
+      const selectedTag = def.selected ? " (SELECTED)" : "";
+      const idTag = isMulti ? ` [id="${def.id}"]` : "";
+
+      systemParts.push(
+        `Interactable component "${name}"${idTag}${selectedTag} (${def.description}). Current state: ${JSON.stringify(def.state)}`,
+      );
+
+      const safeName = name.replace(/[^a-zA-Z0-9_-]/g, "_");
+      const safeId = def.id.replace(/[^a-zA-Z0-9_-]/g, "_");
+      const toolName = isMulti
+        ? `update_${safeName}_${safeId}`
+        : `update_${safeName}`;
+
+      const partialSchema = partialSchemaCache.get(def.id) ?? def.stateSchema;
+
+      tools[toolName] = {
+        type: "frontend" as const,
+        description: `Update the state of interactable component "${name}"${isMulti ? ` (id: ${def.id})` : ""}. Only include the fields you want to change; omitted fields keep their current values. ${def.description}`,
+        parameters: partialSchema,
+        streamCall: async (reader) => {
+          try {
+            for await (const partialArgs of reader.args.streamValues()) {
+              setDefState(def.id, (prev) => shallowMerge(prev, partialArgs));
+            }
+          } catch {
+            // Non-fatal: execute handles the final state
+          }
+        },
+        execute: async (partialState: unknown) => {
+          setDefState(def.id, (prev) => shallowMerge(prev, partialState));
+          return { success: true };
+        },
+      };
+    }
+  }
+
+  return { system: systemParts.join("\n"), tools };
+}

package/src/react/index.ts

@@ -14,6 +14,10 @@ export {
   makeAssistantDataUI,
 } from "./model-context/makeAssistantDataUI";
 export { useAssistantInstructions } from "./model-context/useAssistantInstructions";
+export {
+  useAssistantContext,
+  type AssistantContextConfig,
+} from "./model-context/useAssistantContext";
 export {
   useAssistantTool,
   type AssistantToolProps,
@@ -28,10 +32,20 @@ export {
 } from "./model-context/useAssistantDataUI";
 export { useInlineRender } from "./model-context/useInlineRender";
 export type { Toolkit, ToolDefinition } from "./model-context/toolbox";
+export {
+  useAssistantInteractable,
+  type AssistantInteractableProps,
+} from "./model-context/useAssistantInteractable";
+export { useInteractableState } from "./model-context/useInteractableState";
+export {
+  useToolArgsStatus,
+  type ToolArgsStatus,
+} from "./model-context/useToolArgsStatus";
 
 // client
 export { Tools } from "./client/Tools";
 export { DataRenderers } from "./client/DataRenderers";
+export { Interactables } from "./client/Interactables";
 
 // types
 export type {
@@ -68,6 +82,17 @@ export type {
   DataRenderersMethods,
   DataRenderersClientSchema,
 } from "./types/scopes/dataRenderers";
+export type {
+  InteractableStateSchema,
+  InteractablesState,
+  InteractableDefinition,
+  InteractableRegistration,
+  InteractablesMethods,
+  InteractablePersistedState,
+  InteractablePersistenceAdapter,
+  InteractablePersistenceStatus,
+  InteractablesClientSchema,
+} from "./types/scopes/interactables";
 
 // providers
 export {
@@ -192,6 +217,11 @@ export {
 } from "./primitive-hooks/useActionBarFeedback";
 export { useActionBarSpeak } from "./primitive-hooks/useActionBarSpeak";
 export { useActionBarStopSpeaking } from "./primitive-hooks/useActionBarStopSpeaking";
+export {
+  useVoiceState,
+  useVoiceVolume,
+  useVoiceControls,
+} from "./primitive-hooks/useVoice";
 export { useBranchPickerNext } from "./primitive-hooks/useBranchPickerNext";
 export { useBranchPickerPrevious } from "./primitive-hooks/useBranchPickerPrevious";
 export {

package/src/react/model-context/useAssistantContext.ts

@@ -0,0 +1,22 @@
+import { useEffect, useRef } from "react";
+import { useAui } from "@assistant-ui/store";
+import type { AssistantContextConfig } from "../..";
+
+export type { AssistantContextConfig };
+
+export const useAssistantContext = (config: AssistantContextConfig) => {
+  const { getContext, disabled = false } = config;
+  const aui = useAui();
+  const getContextRef = useRef(getContext);
+  getContextRef.current = getContext;
+
+  useEffect(() => {
+    if (disabled) return;
+
+    return aui.modelContext().register({
+      getModelContext: () => ({
+        system: getContextRef.current(),
+      }),
+    });
+  }, [aui, disabled]);
+};