@assistant-ui/core 0.1.8 → 0.1.10

package/src/react/model-context/useAssistantInteractable.ts

@@ -0,0 +1,47 @@
+import { useEffect, useId, useRef } from "react";
+import { useAui } from "@assistant-ui/store";
+import type { InteractableStateSchema } from "../types/scopes/interactables";
+
+export type AssistantInteractableProps = {
+  description: string;
+  stateSchema: InteractableStateSchema;
+  initialState: unknown;
+  id?: string;
+  selected?: boolean;
+};
+
+/**
+ * Registers an interactable with the AI assistant.
+ *
+ * This hook handles registration only. To read and write the interactable's
+ * state, use {@link useInteractableState} with the returned id.
+ *
+ * @returns The interactable instance id.
+ */
+export const useAssistantInteractable = (
+  name: string,
+  config: AssistantInteractableProps,
+): string => {
+  const aui = useAui();
+
+  const autoId = useId().replace(/[^a-zA-Z0-9]/g, "");
+  const id = config.id ?? autoId;
+
+  const stateSchemaRef = useRef(config.stateSchema);
+  stateSchemaRef.current = config.stateSchema;
+  const initialStateRef = useRef(config.initialState);
+  initialStateRef.current = config.initialState;
+
+  useEffect(() => {
+    return aui.interactables().register({
+      id,
+      name,
+      description: config.description,
+      stateSchema: stateSchemaRef.current,
+      initialState: initialStateRef.current,
+      selected: config.selected,
+    });
+  }, [aui, id, name, config.description, config.selected]);
+
+  return id;
+};

package/src/react/model-context/useInteractableState.ts

@@ -0,0 +1,63 @@
+import { useCallback } from "react";
+import { useAui, useAuiState } from "@assistant-ui/store";
+
+type StateUpdater<TState> = TState | ((prev: TState) => TState);
+
+/**
+ * Reads and writes the state of a registered interactable.
+ *
+ * Pair with {@link useAssistantInteractable} which handles registration.
+ */
+export const useInteractableState = <TState>(
+  id: string,
+  fallback: TState,
+): [
+  TState,
+  {
+    setState: (updater: StateUpdater<TState>) => void;
+    setSelected: (selected: boolean) => void;
+    isPending: boolean;
+    error: unknown;
+    flush: () => Promise<void>;
+  },
+] => {
+  const aui = useAui();
+
+  const state =
+    (useAuiState((s) => s.interactables.definitions[id]?.state) as TState) ??
+    (fallback as TState);
+
+  const persistenceStatus = useAuiState((s) => s.interactables.persistence[id]);
+
+  const setState = useCallback(
+    (updater: StateUpdater<TState>) => {
+      aui.interactables().setState(id, (prev) => {
+        if (typeof updater === "function") {
+          return (updater as (prev: TState) => TState)(prev as TState);
+        }
+        return updater;
+      });
+    },
+    [aui, id],
+  );
+
+  const setSelected = useCallback(
+    (selected: boolean) => {
+      aui.interactables().setSelected(id, selected);
+    },
+    [aui, id],
+  );
+
+  const flush = useCallback(() => aui.interactables().flush(), [aui]);
+
+  return [
+    state,
+    {
+      setState,
+      setSelected,
+      isPending: persistenceStatus?.isPending ?? false,
+      error: persistenceStatus?.error,
+      flush,
+    },
+  ];
+};

package/src/react/model-context/useToolArgsStatus.ts

@@ -0,0 +1,51 @@
+import { useMemo } from "react";
+import { useAuiState } from "@assistant-ui/store";
+import {
+  getPartialJsonObjectFieldState,
+  getPartialJsonObjectMeta,
+} from "assistant-stream/utils";
+
+type PropFieldStatus = "streaming" | "complete";
+
+export type ToolArgsStatus<
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  status: "running" | "complete" | "incomplete" | "requires-action";
+  propStatus: Partial<Record<keyof TArgs, PropFieldStatus>>;
+};
+
+export const useToolArgsStatus = <
+  TArgs extends Record<string, unknown> = Record<string, unknown>,
+>(): ToolArgsStatus<TArgs> => {
+  const part = useAuiState((s) => s.part);
+
+  return useMemo(() => {
+    const statusType = part.status.type;
+
+    if (part.type !== "tool-call") {
+      throw new Error(
+        "useToolArgsStatus can only be used inside tool-call message parts",
+      );
+    }
+
+    const isStreaming = statusType === "running";
+    const args = part.args as Record<string, unknown>;
+    const meta = getPartialJsonObjectMeta(args as Record<symbol, unknown>);
+    const propStatus: Partial<Record<string, PropFieldStatus>> = {};
+
+    for (const key of Object.keys(args)) {
+      if (meta) {
+        const fieldState = getPartialJsonObjectFieldState(args, [key]);
+        propStatus[key] =
+          fieldState === "complete" || !isStreaming ? "complete" : "streaming";
+      } else {
+        propStatus[key] = isStreaming ? "streaming" : "complete";
+      }
+    }
+
+    return {
+      status: statusType,
+      propStatus: propStatus as Partial<Record<keyof TArgs, PropFieldStatus>>,
+    };
+  }, [part]);
+};

package/src/react/primitive-hooks/useVoice.ts

@@ -0,0 +1,41 @@
+import { useCallback, useSyncExternalStore } from "react";
+import { useAui, useAuiState } from "@assistant-ui/store";
+import type { VoiceSessionState } from "../../runtime/interfaces/thread-runtime-core";
+
+export const useVoiceState = (): VoiceSessionState | undefined => {
+  return useAuiState((s) => s.thread.voice);
+};
+
+const getServerVolume = () => 0;
+
+export const useVoiceVolume = (): number => {
+  const aui = useAui();
+  const thread = aui.thread();
+  return useSyncExternalStore(
+    thread.subscribeVoiceVolume,
+    thread.getVoiceVolume,
+    getServerVolume,
+  );
+};
+
+export const useVoiceControls = () => {
+  const aui = useAui();
+
+  const connect = useCallback(() => {
+    aui.thread().connectVoice();
+  }, [aui]);
+
+  const disconnect = useCallback(() => {
+    aui.thread().disconnectVoice();
+  }, [aui]);
+
+  const mute = useCallback(() => {
+    aui.thread().muteVoice();
+  }, [aui]);
+
+  const unmute = useCallback(() => {
+    aui.thread().unmuteVoice();
+  }, [aui]);
+
+  return { connect, disconnect, mute, unmute };
+};

package/src/react/primitives/message/MessageParts.tsx

@@ -440,6 +440,8 @@ const EmptyPartsImpl: FC<MessagePartComponentProps> = ({ components }) => {
 
   if (components?.Empty) return <components.Empty status={status} />;
 
+  if (status.type !== "running") return null;
+
   return (
     <EmptyPartFallback
       status={status}

package/src/react/types/scopes/interactables.ts

@@ -0,0 +1,66 @@
+import type { Tool } from "assistant-stream";
+import type { Unsubscribe } from "../../..";
+
+/**
+ * Schema type matching Tool["parameters"] from assistant-stream.
+ * Accepts both StandardSchemaV1 and JSONSchema7.
+ */
+export type InteractableStateSchema = NonNullable<
+  Extract<Tool, { parameters: unknown }>["parameters"]
+>;
+
+export type InteractableDefinition = {
+  id: string;
+  name: string;
+  description: string;
+  stateSchema: InteractableStateSchema;
+  state: unknown;
+  selected?: boolean | undefined;
+};
+
+export type InteractableRegistration = {
+  id: string;
+  name: string;
+  description: string;
+  stateSchema: InteractableStateSchema;
+  initialState: unknown;
+  selected?: boolean | undefined;
+};
+
+export type InteractablePersistenceStatus = {
+  isPending: boolean;
+  error: unknown;
+};
+
+export type InteractablesState = {
+  /** Keyed by instance id */
+  definitions: Record<string, InteractableDefinition>;
+  /** Per-id persistence sync status */
+  persistence: Record<string, InteractablePersistenceStatus>;
+};
+
+export type InteractablePersistedState = Record<
+  string,
+  { name: string; state: unknown }
+>;
+
+export type InteractablePersistenceAdapter = {
+  save(state: InteractablePersistedState): void | Promise<void>;
+};
+
+export type InteractablesMethods = {
+  getState(): InteractablesState;
+  register(def: InteractableRegistration): Unsubscribe;
+  setState(id: string, updater: (prev: unknown) => unknown): void;
+  setSelected(id: string, selected: boolean): void;
+  exportState(): InteractablePersistedState;
+  importState(saved: InteractablePersistedState): void;
+  setPersistenceAdapter(
+    adapter: InteractablePersistenceAdapter | undefined,
+  ): void;
+  flush(): Promise<void>;
+};
+
+export type InteractablesClientSchema = {
+  methods: InteractablesMethods;
+};

package/src/react/types/store-augmentation.ts

@@ -1,9 +1,11 @@
 import type { ToolsClientSchema } from "./scopes/tools";
 import type { DataRenderersClientSchema } from "./scopes/dataRenderers";
+import type { InteractablesClientSchema } from "./scopes/interactables";
 
 declare module "@assistant-ui/store" {
   interface ScopeRegistry {
     tools: ToolsClientSchema;
     dataRenderers: DataRenderersClientSchema;
+    interactables: InteractablesClientSchema;
   }
 }

package/src/runtime/api/thread-runtime.ts

@@ -3,6 +3,7 @@ import {
   RuntimeCapabilities,
   ThreadRuntimeCore,
   SpeechState,
+  VoiceSessionState,
   ThreadRuntimeEventType,
   StartRunConfig,
   ResumeRunConfig,
@@ -187,6 +188,8 @@ export type ThreadState = {
    * @deprecated This API is still under active development and might change without notice.
    */
   readonly speech: SpeechState | undefined;
+
+  readonly voice: VoiceSessionState | undefined;
 };
 
 export const getThreadState = (
@@ -209,6 +212,7 @@ export const getThreadState = (
     suggestions: runtime.suggestions,
     extras: runtime.extras,
     speech: runtime.speech,
+    voice: runtime.voice,
   });
 };
 
@@ -317,6 +321,13 @@ export type ThreadRuntime = {
    */
   stopSpeaking(): void;
 
+  connectVoice(): void;
+  disconnectVoice(): void;
+  getVoiceVolume(): number;
+  subscribeVoiceVolume(callback: () => void): Unsubscribe;
+  muteVoice(): void;
+  unmuteVoice(): void;
+
   unstable_on(event: ThreadRuntimeEventType, callback: () => void): Unsubscribe;
 };
 
@@ -388,6 +399,12 @@ export class ThreadRuntimeImpl implements ThreadRuntime {
     this.startRun = this.startRun.bind(this);
     this.cancelRun = this.cancelRun.bind(this);
     this.stopSpeaking = this.stopSpeaking.bind(this);
+    this.connectVoice = this.connectVoice.bind(this);
+    this.disconnectVoice = this.disconnectVoice.bind(this);
+    this.muteVoice = this.muteVoice.bind(this);
+    this.unmuteVoice = this.unmuteVoice.bind(this);
+    this.getVoiceVolume = this.getVoiceVolume.bind(this);
+    this.subscribeVoiceVolume = this.subscribeVoiceVolume.bind(this);
     this.export = this.export.bind(this);
     this.import = this.import.bind(this);
     this.reset = this.reset.bind(this);
@@ -463,6 +480,30 @@ export class ThreadRuntimeImpl implements ThreadRuntime {
     return this._threadBinding.getState().stopSpeaking();
   }
 
+  public connectVoice() {
+    this._threadBinding.getState().connectVoice();
+  }
+
+  public disconnectVoice() {
+    this._threadBinding.getState().disconnectVoice();
+  }
+
+  public getVoiceVolume() {
+    return this._threadBinding.getState().getVoiceVolume();
+  }
+
+  public subscribeVoiceVolume(callback: () => void) {
+    return this._threadBinding.getState().subscribeVoiceVolume(callback);
+  }
+
+  public muteVoice() {
+    this._threadBinding.getState().muteVoice();
+  }
+
+  public unmuteVoice() {
+    this._threadBinding.getState().unmuteVoice();
+  }
+
   public export() {
     return this._threadBinding.getState().export();
   }

package/src/runtime/base/base-composer-runtime-core.ts

@@ -9,7 +9,10 @@ import type { QuoteInfo } from "../../types/quote";
 import type { Unsubscribe } from "../../types/unsubscribe";
 import type { RunConfig } from "../../types/message";
 import { BaseSubscribable } from "../../subscribable/subscribable";
-import type { AttachmentAdapter } from "../../adapters/attachment";
+import {
+  type AttachmentAdapter,
+  fileMatchesAccept,
+} from "../../adapters/attachment";
 import type {
   ComposerRuntimeCore,
   ComposerRuntimeEventType,
@@ -216,6 +219,17 @@ export abstract class BaseComposerRuntimeCore
     const adapter = this.getAttachmentAdapter();
     if (!adapter) throw new Error("Attachments are not supported");
 
+    if (
+      !fileMatchesAccept(
+        { name: fileOrAttachment.name, type: fileOrAttachment.type },
+        adapter.accept,
+      )
+    ) {
+      throw new Error(
+        `File type ${fileOrAttachment.type || "unknown"} is not accepted. Accepted types: ${adapter.accept}`,
+      );
+    }
+
     const upsertAttachment = (a: PendingAttachment) => {
       const idx = this._attachments.findIndex(
         (attachment) => attachment.id === a.id,
@@ -233,17 +247,39 @@ export abstract class BaseComposerRuntimeCore
       this._notifySubscribers();
     };
 
-    const promiseOrGenerator = adapter.add({ file: fileOrAttachment });
-    if (Symbol.asyncIterator in promiseOrGenerator) {
-      for await (const r of promiseOrGenerator) {
-        upsertAttachment(r);
+    let lastAttachment: PendingAttachment | undefined;
+    try {
+      const promiseOrGenerator = adapter.add({ file: fileOrAttachment });
+      if (Symbol.asyncIterator in promiseOrGenerator) {
+        for await (const r of promiseOrGenerator) {
+          lastAttachment = r;
+          upsertAttachment(r);
+        }
+      } else {
+        lastAttachment = await promiseOrGenerator;
+        upsertAttachment(lastAttachment);
       }
-    } else {
-      upsertAttachment(await promiseOrGenerator);
+    } catch (e) {
+      if (lastAttachment) {
+        upsertAttachment({
+          ...lastAttachment,
+          status: { type: "incomplete", reason: "error" },
+        });
+      }
+      try {
+        this._notifyEventSubscribers("attachmentAddError");
+      } catch {
+        // prevent subscriber errors from masking the adapter error
+      }
+      throw e;
     }
 
-    this._notifyEventSubscribers("attachmentAdd");
-    this._notifySubscribers();
+    const hasError =
+      lastAttachment?.status.type === "incomplete" &&
+      lastAttachment.status.reason === "error";
+    this._notifyEventSubscribers(
+      hasError ? "attachmentAddError" : "attachmentAdd",
+    );
   }
 
   async removeAttachment(attachmentId: string) {