@assistant-ui/core 0.3.11 → 0.3.13

package/src/tests/external-thread-speech.test.tsx

@@ -0,0 +1,328 @@
+// @vitest-environment jsdom
+
+import { act, render, waitFor } from "@testing-library/react";
+import type { FC } from "react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { AuiProvider, useAui } from "@assistant-ui/store";
+import type { SpeechSynthesisAdapter } from "../index";
+import type {
+  ExternalThreadMessage,
+  ExternalThreadProps,
+} from "../store/clients/external-thread";
+import { ExternalThread } from "../store/clients/external-thread";
+
+const MESSAGES = [
+  {
+    id: "u1",
+    role: "user",
+    content: [{ type: "text", text: "hi" }],
+    createdAt: new Date(0),
+    attachments: [],
+    metadata: { custom: {} },
+  },
+  {
+    id: "a1",
+    role: "assistant",
+    content: [{ type: "text", text: "hello there" }],
+    createdAt: new Date(0),
+    metadata: { custom: {} },
+  },
+] as unknown as readonly ExternalThreadMessage[];
+
+type FakeUtterance = {
+  text: string;
+  cancel: ReturnType<typeof vi.fn>;
+  emit: (status: SpeechSynthesisAdapter.Status) => void;
+};
+
+const createFakeAdapter = () => {
+  const utterances: FakeUtterance[] = [];
+  const adapter: SpeechSynthesisAdapter = {
+    speak: (text) => {
+      const subscribers = new Set<() => void>();
+      // mirror WebSpeechSynthesisAdapter: cancel transitions to ended and notifies synchronously
+      const cancel = vi.fn(() => {
+        if (utterance.status.type === "ended") return;
+        utterance.status = { type: "ended", reason: "cancelled" };
+        for (const subscriber of subscribers) subscriber();
+      });
+      const utterance: SpeechSynthesisAdapter.Utterance = {
+        status: { type: "starting" },
+        cancel,
+        subscribe: (callback) => {
+          subscribers.add(callback);
+          return () => subscribers.delete(callback);
+        },
+      };
+      utterances.push({
+        text,
+        cancel,
+        emit: (status) => {
+          utterance.status = status;
+          for (const subscriber of subscribers) subscriber();
+        },
+      });
+      return utterance;
+    },
+  };
+  return { adapter, utterances };
+};
+
+const renderThreadWithProps = (props: Partial<ExternalThreadProps>) => {
+  const captured: { aui?: ReturnType<typeof useAui> } = {};
+  const Capture: FC = () => {
+    captured.aui = useAui();
+    return null;
+  };
+  const App: FC<{ props: Partial<ExternalThreadProps> }> = ({ props }) => {
+    const aui = useAui({
+      thread: ExternalThread({
+        messages: MESSAGES,
+        isRunning: false,
+        ...props,
+      }),
+    });
+    return (
+      <AuiProvider value={aui}>
+        <Capture />
+      </AuiProvider>
+    );
+  };
+
+  const view = render(<App props={props} />);
+  const aui = () => captured.aui!;
+  aui.rerender = (nextProps: Partial<ExternalThreadProps>) =>
+    view.rerender(<App props={nextProps} />);
+  aui.unmount = () => view.unmount();
+  return aui;
+};
+
+describe("ExternalThread speech", () => {
+  it("reports the speech capability based on adapter presence", () => {
+    const withoutAdapter = renderThreadWithProps({});
+    expect(withoutAdapter().thread.getState().capabilities.speech).toBe(false);
+
+    const { adapter } = createFakeAdapter();
+    const withAdapter = renderThreadWithProps({ speechAdapter: adapter });
+    expect(withAdapter().thread.getState().capabilities.speech).toBe(true);
+  });
+
+  it("throws on speak when no adapter is configured", () => {
+    const aui = renderThreadWithProps({});
+    expect(() => aui().thread.message({ id: "a1" }).speak()).toThrow(
+      "Speech adapter not configured",
+    );
+  });
+
+  it("tracks utterance status in thread and message state", async () => {
+    const { adapter, utterances } = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+
+    expect(utterances[0]!.text).toBe("hello there");
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toEqual({
+        messageId: "a1",
+        status: { type: "starting" },
+      });
+      expect(aui().thread.message({ id: "a1" }).getState().speech).toEqual({
+        messageId: "a1",
+        status: { type: "starting" },
+      });
+      expect(
+        aui().thread.message({ id: "u1" }).getState().speech,
+      ).toBeUndefined();
+    });
+
+    await act(async () => {
+      utterances[0]!.emit({ type: "running" });
+    });
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toEqual({
+        messageId: "a1",
+        status: { type: "running" },
+      });
+    });
+
+    await act(async () => {
+      utterances[0]!.emit({ type: "ended", reason: "finished" });
+    });
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toBeUndefined();
+      expect(
+        aui().thread.message({ id: "a1" }).getState().speech,
+      ).toBeUndefined();
+    });
+  });
+
+  it("cancels the previous utterance when a new speak starts", async () => {
+    const { adapter, utterances } = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+    await act(async () => {
+      aui().thread.message({ id: "u1" }).speak();
+    });
+
+    expect(utterances).toHaveLength(2);
+    expect(utterances[0]!.cancel).toHaveBeenCalledTimes(1);
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toEqual({
+        messageId: "u1",
+        status: { type: "starting" },
+      });
+    });
+
+    await act(async () => {
+      utterances[0]!.emit({ type: "ended", reason: "cancelled" });
+    });
+    expect(aui().thread.getState().speech).toEqual({
+      messageId: "u1",
+      status: { type: "starting" },
+    });
+  });
+
+  it("stopSpeaking cancels the utterance and clears state", async () => {
+    const { adapter, utterances } = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+    await act(async () => {
+      aui().thread.stopSpeaking();
+    });
+
+    expect(utterances[0]!.cancel).toHaveBeenCalledTimes(1);
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toBeUndefined();
+    });
+    expect(() => aui().thread.stopSpeaking()).toThrow(
+      "No message is being spoken",
+    );
+  });
+
+  it("message stopSpeaking throws unless that message is being spoken", async () => {
+    const { adapter, utterances } = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    expect(() => aui().thread.message({ id: "a1" }).stopSpeaking()).toThrow(
+      "Message is not being spoken",
+    );
+
+    // speak and stopSpeaking in the same tick, before any re-render
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+      expect(() => aui().thread.message({ id: "u1" }).stopSpeaking()).toThrow(
+        "Message is not being spoken",
+      );
+      aui().thread.message({ id: "a1" }).stopSpeaking();
+    });
+
+    expect(utterances[0]!.cancel).toHaveBeenCalledTimes(1);
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toBeUndefined();
+    });
+  });
+
+  it("cancels the utterance when the adapter is removed", async () => {
+    const { adapter, utterances } = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+    await act(async () => {
+      aui.rerender({});
+    });
+
+    expect(utterances[0]!.cancel).toHaveBeenCalledTimes(1);
+    expect(aui().thread.getState().capabilities.speech).toBe(false);
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toBeUndefined();
+    });
+
+    await act(async () => {
+      utterances[0]!.emit({ type: "running" });
+    });
+    expect(aui().thread.getState().speech).toBeUndefined();
+  });
+
+  it("keeps the utterance alive when the adapter identity changes", async () => {
+    const first = createFakeAdapter();
+    const second = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: first.adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+    await act(async () => {
+      aui.rerender({ speechAdapter: second.adapter });
+    });
+
+    expect(first.utterances[0]!.cancel).not.toHaveBeenCalled();
+    expect(aui().thread.getState().speech).toEqual({
+      messageId: "a1",
+      status: { type: "starting" },
+    });
+
+    await act(async () => {
+      aui().thread.message({ id: "u1" }).speak();
+    });
+    expect(first.utterances[0]!.cancel).toHaveBeenCalledTimes(1);
+    expect(second.utterances[0]!.text).toBe("hi");
+    await waitFor(() => {
+      expect(aui().thread.getState().speech).toEqual({
+        messageId: "u1",
+        status: { type: "starting" },
+      });
+    });
+  });
+
+  it("handles utterances that end synchronously on subscribe", async () => {
+    const cancel = vi.fn();
+    const adapter: SpeechSynthesisAdapter = {
+      speak: () => ({
+        status: { type: "ended", reason: "finished" },
+        cancel,
+        subscribe: (callback) => {
+          callback();
+          return () => {};
+        },
+      }),
+    };
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+
+    expect(aui().thread.getState().speech).toBeUndefined();
+    expect(() => aui().thread.stopSpeaking()).toThrow(
+      "No message is being spoken",
+    );
+  });
+
+  it("cancels the utterance on unmount", async () => {
+    const { adapter, utterances } = createFakeAdapter();
+    const aui = renderThreadWithProps({ speechAdapter: adapter });
+
+    await act(async () => {
+      aui().thread.message({ id: "a1" }).speak();
+    });
+    await act(async () => {
+      aui.unmount();
+    });
+
+    expect(utterances[0]!.cancel).toHaveBeenCalledTimes(1);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+});

package/src/tests/external-thread-suggestions.test.tsx

@@ -0,0 +1,59 @@
+// @vitest-environment jsdom
+
+import { cleanup, render } from "@testing-library/react";
+import type { FC, ReactNode } from "react";
+import { afterEach, describe, expect, it } from "vitest";
+import { AuiProvider, useAui } from "@assistant-ui/store";
+import { ExternalThread } from "../store/clients/external-thread";
+import { SingleThreadList } from "../store/clients/single-thread-list";
+
+let childAui!: ReturnType<typeof useAui>;
+
+const CaptureChild: FC = () => {
+  childAui = useAui();
+  return null;
+};
+
+const Child: FC = () => {
+  const aui = useAui({ thread: ExternalThread({ messages: [] }) });
+  return (
+    <AuiProvider value={aui}>
+      <CaptureChild />
+    </AuiProvider>
+  );
+};
+
+const Parent: FC<{ children: ReactNode }> = ({ children }) => {
+  const aui = useAui({
+    threads: SingleThreadList({ thread: ExternalThread({ messages: [] }) }),
+  });
+  return <AuiProvider value={aui}>{children}</AuiProvider>;
+};
+
+afterEach(() => {
+  cleanup();
+});
+
+describe("ExternalThread suggestions scope", () => {
+  it("derives the scope from its own thread when standalone", () => {
+    render(<Child />);
+
+    expect(childAui.suggestions.getState()).toEqual({ suggestions: [] });
+    expect(childAui.suggestions.getState()).toBe(
+      childAui.thread.suggestions().getState(),
+    );
+  });
+
+  it("derives the scope from its own thread when nested under a threads parent", () => {
+    render(
+      <Parent>
+        <Child />
+      </Parent>,
+    );
+
+    expect(childAui.suggestions.getState()).toEqual({ suggestions: [] });
+    expect(childAui.suggestions.getState()).toBe(
+      childAui.thread.suggestions().getState(),
+    );
+  });
+});

package/src/tests/message-queue.test.ts

@@ -626,3 +626,29 @@ describe("createMessageQueue", () => {
     expect(adapter.items).toHaveLength(0);
   });
 });
+
+describe("createMessageQueue interrupt with a runtime-routed cancel", () => {
+  it("keeps draining when the driver's cancel notifies back", () => {
+    const runs: string[] = [];
+    const controller = createMessageQueue({
+      run: (message) => {
+        runs.push(
+          message.content[0]!.type === "text" ? message.content[0].text : "",
+        );
+      },
+      cancel: () => controller.notifyCancelled(),
+    });
+
+    controller.adapter.enqueue(msg("first"));
+    controller.adapter.enqueue(msg("second"));
+    controller.adapter.steer(msg("steered"));
+
+    expect(runs).toEqual(["first", "steered"]);
+
+    // the interrupted run and the steer run each settle once
+    controller.notifyIdle();
+    controller.notifyIdle();
+
+    expect(runs).toEqual(["first", "steered", "second"]);
+  });
+});

package/src/tests/thread-message-like.test.ts

@@ -382,3 +382,42 @@ describe("fromThreadMessageLike", () => {
     });
   });
 });
+
+describe("fromThreadMessageLike provider metadata", () => {
+  it("keeps it on image and file parts", () => {
+    const result = fromThreadMessageLike(
+      {
+        role: "user",
+        content: [
+          {
+            type: "image",
+            image: "https://example.com/cat.png",
+            providerMetadata: { agui: { file_id: "f_1" } },
+          },
+          {
+            type: "file",
+            data: "https://example.com/spec.pdf",
+            mimeType: "application/pdf",
+            providerMetadata: { agui: { file_id: "f_2" } },
+          },
+        ],
+      },
+      fallbackId,
+      fallbackStatus,
+    );
+
+    expect(result.content).toEqual([
+      {
+        type: "image",
+        image: "https://example.com/cat.png",
+        providerMetadata: { agui: { file_id: "f_1" } },
+      },
+      {
+        type: "file",
+        data: "https://example.com/spec.pdf",
+        mimeType: "application/pdf",
+        providerMetadata: { agui: { file_id: "f_2" } },
+      },
+    ]);
+  });
+});

package/src/tests/thread-switch-events.test.tsx

@@ -0,0 +1,136 @@
+// @vitest-environment jsdom
+
+import { act, render } from "@testing-library/react";
+import { describe, expect, it, vi } from "vitest";
+import { AuiProvider, useAui, useAuiEvent } from "@assistant-ui/store";
+import { RuntimeAdapter } from "../react/RuntimeAdapter";
+import { AssistantRuntimeImpl } from "../runtime/api/assistant-runtime";
+import { ExternalStoreRuntimeCore } from "../runtimes/external-store/external-store-runtime-core";
+import type { ExternalStoreAdapter } from "../runtimes/external-store/external-store-adapter";
+
+type DemoMessage = { id: string; role: "user" | "assistant"; text: string };
+
+const createRuntime = () => {
+  const threads = [
+    {
+      id: "t1",
+      title: "one",
+      messages: [{ id: "m1", role: "user" as const, text: "a" }],
+    },
+    {
+      id: "t2",
+      title: "two",
+      messages: [{ id: "m2", role: "user" as const, text: "b" }],
+    },
+  ];
+  let currentId = "t1";
+  const makeAdapter = (): ExternalStoreAdapter<DemoMessage> => ({
+    messages: threads.find((t) => t.id === currentId)!.messages,
+    convertMessage: (m) => ({
+      id: m.id,
+      role: m.role,
+      content: [{ type: "text", text: m.text }],
+    }),
+    onNew: async () => {},
+    adapters: {
+      threadList: {
+        threadId: currentId,
+        threads: threads.map((t) => ({
+          status: "regular" as const,
+          id: t.id,
+          title: t.title,
+        })),
+        onSwitchToThread: (threadId: string) => {
+          currentId = threadId;
+          sync();
+        },
+        onSwitchToNewThread: () => {},
+      },
+    },
+  });
+  const core = new ExternalStoreRuntimeCore(makeAdapter());
+  const runtime = new AssistantRuntimeImpl(core);
+  const sync = () => core.setAdapter(makeAdapter());
+  return runtime;
+};
+
+describe("thread switch events", () => {
+  it("delivers switchedTo to default-scope, star-scope, and aui.on listeners", async () => {
+    const runtime = createRuntime();
+    const defaultScope = vi.fn();
+    const starScope = vi.fn();
+    const auiOn = vi.fn();
+    const switchedAwayStar = vi.fn();
+    let aui!: ReturnType<typeof useAui>;
+    const Consumer = () => {
+      useAuiEvent("threadListItem.switchedTo" as never, defaultScope as never);
+      useAuiEvent(
+        { scope: "*", event: "threadListItem.switchedTo" } as never,
+        starScope as never,
+      );
+      useAuiEvent(
+        { scope: "*", event: "threadListItem.switchedAway" } as never,
+        switchedAwayStar as never,
+      );
+      return null;
+    };
+    const Harness = () => {
+      aui = useAui({ threads: RuntimeAdapter(runtime) } as never);
+      return (
+        <AuiProvider value={aui}>
+          <Consumer />
+        </AuiProvider>
+      );
+    };
+    render(<Harness />);
+    await act(async () => {});
+
+    aui.on("threadListItem.switchedTo" as never, auiOn as never);
+
+    await act(async () => {
+      aui.threads.item({ index: 1 }).switchTo();
+    });
+    await act(async () => {});
+
+    expect(defaultScope).toHaveBeenCalledExactlyOnceWith({ threadId: "t2" });
+    expect(starScope).toHaveBeenCalledExactlyOnceWith({ threadId: "t2" });
+    expect(auiOn).toHaveBeenCalledExactlyOnceWith({ threadId: "t2" });
+    expect(switchedAwayStar).toHaveBeenCalledExactlyOnceWith({
+      threadId: "t1",
+    });
+
+    await act(async () => {
+      aui.threads.item({ index: 0 }).switchTo();
+    });
+    await act(async () => {});
+
+    expect(defaultScope).toHaveBeenCalledTimes(2);
+    expect(defaultScope).toHaveBeenLastCalledWith({ threadId: "t1" });
+    expect(switchedAwayStar).toHaveBeenCalledTimes(2);
+    expect(switchedAwayStar).toHaveBeenLastCalledWith({ threadId: "t2" });
+  });
+
+  it("does not emit for the initially selected thread on mount", async () => {
+    const runtime = createRuntime();
+    const anySwitch = vi.fn();
+    const Consumer = () => {
+      useAuiEvent(
+        { scope: "*", event: "threadListItem.switchedTo" } as never,
+        anySwitch as never,
+      );
+      return null;
+    };
+    const Harness = () => {
+      const aui = useAui({ threads: RuntimeAdapter(runtime) } as never);
+      return (
+        <AuiProvider value={aui}>
+          <Consumer />
+        </AuiProvider>
+      );
+    };
+    render(<Harness />);
+    await act(async () => {});
+
+    expect(anySwitch).not.toHaveBeenCalled();
+  });
+});

package/src/types/augmentations.ts

@@ -0,0 +1,33 @@
+/**
+ * Module augmentation namespace for assistant-ui type extensions.
+ *
+ * @example
+ * ```typescript
+ * declare module "@assistant-ui/core" {
+ *   namespace Assistant {
+ *     interface Commands {
+ *       myCustomCommand: {
+ *         type: "my-custom-command";
+ *         data: string;
+ *       };
+ *     }
+ *
+ *     interface ExternalState {
+ *       myCustomState: {
+ *         foo: string;
+ *       };
+ *     }
+ *   }
+ * }
+ * ```
+ */
+export namespace Assistant {
+  export interface Commands {}
+
+  export interface ExternalState {}
+}
+
+export type UserCommands = Assistant.Commands[keyof Assistant.Commands];
+export type UserExternalState = keyof Assistant.ExternalState extends never
+  ? Record<string, unknown>
+  : Assistant.ExternalState[keyof Assistant.ExternalState];

package/src/types/error.ts

@@ -52,6 +52,30 @@ export const isAssistantError = (value: unknown): value is AssistantError => {
   return true;
 };
 
+const MESSAGE_NOT_SENT = Symbol.for("assistant-ui.message-not-sent");
+
+/**
+ * Rejection reason for a send that never reached the backend, so nothing ran
+ * and nothing is recoverable from the thread. A runtime adapter throws it from
+ * `onNew` to hand the message back to the thread composer, which restores the
+ * draft it cleared at dispatch time when nothing has claimed the composer
+ * since. An edit composer closes at dispatch, so a rejected edit is not
+ * restored.
+ */
+export class MessageNotSentError extends Error {
+  readonly [MESSAGE_NOT_SENT] = true;
+
+  constructor(message = "The message was not sent.") {
+    super(message);
+    this.name = "MessageNotSentError";
+  }
+}
+
+export const isMessageNotSentError = (
+  error: unknown,
+): error is MessageNotSentError =>
+  typeof error === "object" && error !== null && MESSAGE_NOT_SENT in error;
+
 export const toAssistantError = (error: unknown): AssistantError => {
   if (isAssistantError(error)) return error;
   if (error instanceof Error) {

package/src/types/message.ts

@@ -56,6 +56,7 @@ export type ImageMessagePart = {
   readonly type: "image";
   readonly image: string;
   readonly filename?: string;
+  readonly providerMetadata?: PartProviderMetadata;
 };
 
 export type FileMessagePart = {
@@ -65,6 +66,7 @@ export type FileMessagePart = {
   readonly mimeType: string;
   /** How `data` goes on the wire: a url or id reference; omitted = inferred (http(s) → url, else base64). "url" is honored by the LangChain-family, A2A, AG-UI, and Google ADK runtimes; "id" by the LangChain family only. */
   readonly sourceType?: "url" | "id";
+  readonly providerMetadata?: PartProviderMetadata;
   readonly parentId?: string;
 };