@assistant-ui/core 0.3.17 → 0.3.19

package/src/adapters/speech.test.ts

@@ -7,6 +7,62 @@ afterEach(() => {
 });
 
 describe("WebSpeechSynthesisAdapter", () => {
+  const stubSpeechSynthesis = () => {
+    const utterances: EventTarget[] = [];
+    class MockSpeechSynthesisUtterance extends EventTarget {
+      constructor() {
+        super();
+        utterances.push(this);
+      }
+    }
+    const cancel = vi.fn();
+    vi.stubGlobal("SpeechSynthesisUtterance", MockSpeechSynthesisUtterance);
+    vi.stubGlobal("window", { speechSynthesis: { speak: vi.fn(), cancel } });
+    return { utterances, cancel };
+  };
+
+  it.each(["end", "error"])(
+    "does not cancel newer playback through a handle that received %s",
+    (event) => {
+      const { utterances, cancel } = stubSpeechSynthesis();
+      const adapter = new WebSpeechSynthesisAdapter();
+      const old = adapter.speak("old");
+      utterances[0]!.dispatchEvent(new Event(event));
+      const endedStatus = old.status;
+      const current = adapter.speak("current");
+
+      old.cancel();
+      old.cancel();
+
+      expect(cancel).not.toHaveBeenCalled();
+      expect(old.status).toBe(endedStatus);
+      expect(current.status).toEqual({ type: "running" });
+    },
+  );
+
+  it("cancels active playback once and leaves later playback alone", () => {
+    const { cancel } = stubSpeechSynthesis();
+    const adapter = new WebSpeechSynthesisAdapter();
+    const old = adapter.speak("old");
+    const onChange = vi.fn();
+    old.subscribe(onChange);
+
+    old.cancel();
+
+    expect(cancel).toHaveBeenCalledOnce();
+    expect(old.status).toMatchObject({ type: "ended", reason: "cancelled" });
+    expect(onChange).toHaveBeenCalledOnce();
+
+    const current = adapter.speak("current");
+    old.cancel();
+
+    expect(cancel).toHaveBeenCalledOnce();
+    expect(onChange).toHaveBeenCalledOnce();
+    expect(current.status).toEqual({ type: "running" });
+    current.cancel();
+    expect(cancel).toHaveBeenCalledTimes(2);
+  });
+
   it("isolates a late subscriber that throws after the utterance ended", async () => {
     const listeners = new Map<string, EventListener>();
     class MockSpeechSynthesisUtterance {
@@ -109,6 +165,186 @@ describe("WebSpeechDictationAdapter", () => {
     return listeners;
   };
 
+  const emitResults = (
+    listeners: Map<string, EventListener>,
+    results: [transcript: string, isFinal: boolean][],
+    resultIndex = 0,
+  ) => {
+    listeners.get("result")!({
+      resultIndex,
+      results: results.map(([transcript, isFinal]) => ({
+        0: { transcript },
+        isFinal,
+      })),
+    } as unknown as Event);
+  };
+
+  it("publishes the entire interim suffix when only its last result changes", () => {
+    const listeners = stubSpeechRecognition();
+    const session = new WebSpeechDictationAdapter().listen();
+    const onSpeech = vi.fn();
+    session.onSpeech(onSpeech);
+
+    emitResults(listeners, [
+      ["hello ", false],
+      ["world", false],
+    ]);
+    emitResults(
+      listeners,
+      [
+        ["hello ", false],
+        ["there", false],
+      ],
+      1,
+    );
+
+    expect(onSpeech.mock.calls).toEqual([
+      [{ transcript: "hello world", isFinal: false }],
+      [{ transcript: "hello there", isFinal: false }],
+    ]);
+  });
+
+  it("publishes partial and complete retractions with no changed results", () => {
+    const listeners = stubSpeechRecognition();
+    const session = new WebSpeechDictationAdapter().listen();
+    const onSpeech = vi.fn();
+    session.onSpeech(onSpeech);
+
+    emitResults(listeners, [
+      ["hello ", false],
+      ["world", false],
+    ]);
+    emitResults(listeners, [["hello ", false]], 1);
+    emitResults(listeners, []);
+    emitResults(listeners, []);
+
+    expect(onSpeech.mock.calls).toEqual([
+      [{ transcript: "hello world", isFinal: false }],
+      [{ transcript: "hello ", isFinal: false }],
+      [{ transcript: "", isFinal: false }],
+    ]);
+  });
+
+  it("delivers changed final results once before the remaining interim suffix", () => {
+    const listeners = stubSpeechRecognition();
+    const session = new WebSpeechDictationAdapter().listen();
+    const onSpeech = vi.fn();
+    const onEnd = vi.fn();
+    session.onSpeech(onSpeech);
+    session.onSpeechEnd(onEnd);
+
+    emitResults(listeners, [
+      ["hello ", true],
+      ["wor", false],
+      ["ld", false],
+    ]);
+    emitResults(
+      listeners,
+      [
+        ["hello ", true],
+        ["world", true],
+      ],
+      1,
+    );
+    listeners.get("end")!(new Event("end"));
+
+    expect(onSpeech.mock.calls).toEqual([
+      [{ transcript: "hello ", isFinal: true }],
+      [{ transcript: "world", isFinal: false }],
+      [{ transcript: "world", isFinal: true }],
+    ]);
+    expect(onEnd).toHaveBeenCalledExactlyOnceWith({
+      transcript: "hello world",
+    });
+  });
+
+  it("clears an interim suffix while retaining earlier final results", () => {
+    const listeners = stubSpeechRecognition();
+    const session = new WebSpeechDictationAdapter().listen();
+    const onSpeech = vi.fn();
+    session.onSpeech(onSpeech);
+
+    emitResults(listeners, [
+      ["hello ", true],
+      ["world", false],
+    ]);
+    emitResults(listeners, [["hello ", true]], 1);
+
+    expect(onSpeech.mock.calls).toEqual([
+      [{ transcript: "hello ", isFinal: true }],
+      [{ transcript: "world", isFinal: false }],
+      [{ transcript: "", isFinal: false }],
+    ]);
+  });
+
+  it("keeps final-only delivery free of empty interim updates", () => {
+    const listeners = stubSpeechRecognition();
+    const session = new WebSpeechDictationAdapter().listen();
+    const onSpeech = vi.fn();
+    session.onSpeech(onSpeech);
+
+    emitResults(listeners, [["hello", true]]);
+    emitResults(listeners, [["hello", true]], 1);
+
+    expect(onSpeech).toHaveBeenCalledExactlyOnceWith({
+      transcript: "hello",
+      isFinal: true,
+    });
+  });
+
+  it("does not drop an unconsumed final when resultIndex skips it", () => {
+    const listeners = stubSpeechRecognition();
+    const session = new WebSpeechDictationAdapter().listen();
+    const onSpeech = vi.fn();
+    const onEnd = vi.fn();
+    session.onSpeech(onSpeech);
+    session.onSpeechEnd(onEnd);
+
+    emitResults(listeners, [["hello ", false]]);
+    emitResults(
+      listeners,
+      [
+        ["hello ", true],
+        ["world", false],
+      ],
+      1,
+    );
+    listeners.get("end")!(new Event("end"));
+
+    expect(onSpeech.mock.calls).toEqual([
+      [{ transcript: "hello ", isFinal: false }],
+      [{ transcript: "hello ", isFinal: true }],
+      [{ transcript: "world", isFinal: false }],
+    ]);
+    expect(onEnd).toHaveBeenCalledExactlyOnceWith({ transcript: "hello " });
+  });
+
+  it("does not reread historical finalized results during interim updates", () => {
+    const listeners = stubSpeechRecognition();
+    new WebSpeechDictationAdapter().listen();
+    const readFinal = vi.fn(() => ({ transcript: "word " }));
+    const finals = Array.from({ length: 1_000 }, () => ({
+      get 0() {
+        return readFinal();
+      },
+      isFinal: true,
+    }));
+    listeners.get("result")!({
+      resultIndex: 0,
+      results: finals,
+    } as unknown as Event);
+    expect(readFinal).toHaveBeenCalledTimes(1_000);
+
+    for (const transcript of ["hel", "hello", "hello world"]) {
+      listeners.get("result")!({
+        resultIndex: finals.length,
+        results: [...finals, { 0: { transcript }, isFinal: false }],
+      } as unknown as Event);
+    }
+
+    expect(readFinal).toHaveBeenCalledTimes(1_000);
+  });
+
   it("continues notifying dictation listeners when one throws", () => {
     const listeners = stubSpeechRecognition();
     const listenerError = new Error("listener failed");

package/src/adapters/speech.ts

@@ -76,6 +76,7 @@ export class WebSpeechSynthesisAdapter implements SpeechSynthesisAdapter {
     const res: SpeechSynthesisAdapter.Utterance = {
       status: { type: "running" },
       cancel: () => {
+        if (res.status.type === "ended") return;
         window.speechSynthesis.cancel();
         handleEnd("cancelled");
       },
@@ -205,6 +206,8 @@ export class WebSpeechDictationAdapter implements DictationAdapter {
     >();
 
     let finalTranscript = "";
+    let hasInterimTranscript = false;
+    let firstInterimIndex = 0;
 
     const session: DictationAdapter.Session = {
       status: { type: "starting" },
@@ -267,32 +270,36 @@ export class WebSpeechDictationAdapter implements DictationAdapter {
 
     recognition.addEventListener("result", (event) => {
       const speechEvent = event as unknown as SpeechRecognitionEvent;
+      let interimTranscript = "";
 
-      for (
-        let i = speechEvent.resultIndex;
-        i < speechEvent.results.length;
-        i++
-      ) {
+      for (let i = firstInterimIndex; i < speechEvent.results.length; i++) {
         const result = speechEvent.results[i];
         if (!result) continue;
 
         const transcript = result[0]?.transcript ?? "";
 
         if (result.isFinal) {
+          firstInterimIndex = i + 1;
           finalTranscript += transcript;
+          hasInterimTranscript = false;
           notifyEventListeners(
             speechCallbacks,
             () => ({ transcript, isFinal: true }),
             "Dictation",
           );
         } else {
-          notifyEventListeners(
-            speechCallbacks,
-            () => ({ transcript, isFinal: false }),
-            "Dictation",
-          );
+          interimTranscript += transcript;
         }
       }
+
+      if (interimTranscript || hasInterimTranscript) {
+        hasInterimTranscript = interimTranscript.length > 0;
+        notifyEventListeners(
+          speechCallbacks,
+          () => ({ transcript: interimTranscript, isFinal: false }),
+          "Dictation",
+        );
+      }
     });
 
     recognition.addEventListener("speechend", () => {

package/src/adapters/thread-history.ts

@@ -7,6 +7,7 @@ import type {
   ExportedMessageRepositoryItem,
 } from "../runtime/utils/message-repository";
 import type { ReadonlyJSONValue } from "assistant-stream/utils";
+import type { ThreadMessage } from "../types";
 
 export interface MessageStorageEntry<TPayload> {
   id: string;
@@ -52,6 +53,8 @@ export type GenericThreadHistoryAdapter<TMessage> = {
     options?: {
       durationMs?: number;
       stepTimestamps?: { start_ms: number; end_ms: number }[];
+      /** The thread message the items were persisted from; its status and timing complete a report the stored format cannot carry. */
+      message?: ThreadMessage;
     },
   ): void;
 };

package/src/index.ts

@@ -40,6 +40,7 @@ export type {
   MessageStatus,
   // Thread messages
   MessageTiming,
+  MessageModality,
   ThreadStep,
   ThreadSystemMessage,
   ThreadUserMessage,
@@ -261,6 +262,7 @@ export type {
   ThreadListItemEventCallback,
   ThreadListItemEventPayload,
   ThreadListItemEventType,
+  ThreadListItemGenerateTitleOptions,
   ThreadListItemRuntime,
 } from "./runtime/api/thread-list-item-runtime";
 

package/src/internal.ts

@@ -67,6 +67,10 @@ export {
   resolveFilePartSource,
 } from "./utils/data-url";
 export { invokeUserCallback } from "./utils/invoke-user-callback";
+export {
+  abortableIterable,
+  openAbortableIterable,
+} from "./utils/abortable-iterable";
 export { detectImageMediaType } from "./utils/image-media-type";
 export {
   resolveFileMediaType,

package/src/model-context/frame/host.test.ts

@@ -74,9 +74,31 @@ beforeEach(() => {
 afterEach(() => {
   vi.useRealTimers();
   vi.unstubAllGlobals();
+  vi.restoreAllMocks();
 });
 
 describe("AssistantFrameHost", () => {
+  it("does not install its message listener when initialization fails", () => {
+    const addEventListener = vi.fn();
+    const removeEventListener = vi.fn();
+    vi.stubGlobal("window", {
+      addEventListener,
+      removeEventListener,
+      location: { origin: DEFAULT_ORIGIN },
+    });
+    const error = new Error("postMessage failed");
+    const iframeWindow = {
+      postMessage: vi.fn(() => {
+        throw error;
+      }),
+    } as unknown as Window;
+
+    expect(() => new AssistantFrameHost(iframeWindow)).toThrow(error);
+
+    expect(addEventListener).not.toHaveBeenCalled();
+    expect(removeEventListener).not.toHaveBeenCalled();
+  });
+
   it("defaults to the current origin", () => {
     const { host, postMessage } = createHost();
 
@@ -133,6 +155,30 @@ describe("AssistantFrameHost", () => {
     host.dispose();
   });
 
+  it("cleans up tool calls when posting the request fails", async () => {
+    const { execute, host, postMessage } = createHost();
+    const error = new Error("postMessage failed");
+    const abortController = new AbortController();
+    const removeEventListener = vi.spyOn(
+      abortController.signal,
+      "removeEventListener",
+    );
+    postMessage.mockImplementation((data) => {
+      if (data.message.type === "tool-call") throw error;
+    });
+
+    await expect(
+      execute({}, { ...executionContext, abortSignal: abortController.signal }),
+    ).rejects.toBe(error);
+
+    expect(vi.getTimerCount()).toBe(0);
+    expect(removeEventListener).toHaveBeenCalledWith(
+      "abort",
+      expect.any(Function),
+    );
+    host.dispose();
+  });
+
   it("rejects tool results that carry an empty error message", async () => {
     const { dispatchMessage, execute, getToolCallId, host } = createHost();
     const result = Promise.resolve(
@@ -188,8 +234,83 @@ describe("AssistantFrameHost", () => {
     expect(vi.getTimerCount()).toBe(0);
   });
 
+  it("settles every pending tool call when cancellation posting fails", async () => {
+    const { execute, host, postMessage } = createHost();
+    const consoleError = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => undefined);
+    const firstAbortController = new AbortController();
+    const secondAbortController = new AbortController();
+    const firstRemoveEventListener = vi.spyOn(
+      firstAbortController.signal,
+      "removeEventListener",
+    );
+    const secondRemoveEventListener = vi.spyOn(
+      secondAbortController.signal,
+      "removeEventListener",
+    );
+    const firstRejected = vi.fn();
+    const secondRejected = vi.fn();
+
+    void execute(
+      {},
+      { ...executionContext, abortSignal: firstAbortController.signal },
+    ).catch(firstRejected);
+    void execute(
+      {},
+      { ...executionContext, abortSignal: secondAbortController.signal },
+    ).catch(secondRejected);
+
+    const firstTransportError = new Error("first tool cancellation failed");
+    const secondTransportError = new Error("second tool cancellation failed");
+    let cancellationCount = 0;
+    postMessage.mockImplementation((data) => {
+      if (data.message.type !== "tool-cancel") return;
+      cancellationCount += 1;
+      throw cancellationCount === 1
+        ? firstTransportError
+        : secondTransportError;
+    });
+
+    expect(() => host.dispose()).toThrow(firstTransportError);
+    await Promise.resolve();
+
+    expect(firstRejected).toHaveBeenCalledWith(
+      expect.objectContaining({
+        message: "AssistantFrameHost has been disposed",
+      }),
+    );
+    expect(secondRejected).toHaveBeenCalledWith(
+      expect.objectContaining({
+        message: "AssistantFrameHost has been disposed",
+      }),
+    );
+    expect(
+      postMessage.mock.calls.filter(
+        ([data]) => data.message.type === "tool-cancel",
+      ),
+    ).toHaveLength(2);
+    expect(firstRemoveEventListener).toHaveBeenCalledWith(
+      "abort",
+      expect.any(Function),
+    );
+    expect(secondRemoveEventListener).toHaveBeenCalledWith(
+      "abort",
+      expect.any(Function),
+    );
+    expect(vi.getTimerCount()).toBe(0);
+    expect(consoleError).toHaveBeenCalledWith(
+      "[assistant-ui] AssistantFrameHost tool cancellation could not be sent.",
+      secondTransportError,
+    );
+    expect(() => host.dispose()).not.toThrow();
+  });
+
   it("rejects pending tool calls when execution is aborted", async () => {
     const { execute, getToolCallId, host, postMessage } = createHost();
+    const consoleError = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => undefined);
     const abortController = new AbortController();
     const abortError = new Error("Run cancelled");
     abortError.name = "AbortError";
@@ -205,8 +326,13 @@ describe("AssistantFrameHost", () => {
     const onRejected = vi.fn();
     void result.catch(onRejected);
     const toolCallId = getToolCallId();
+    postMessage.mockImplementation((data) => {
+      if (data.message.type === "tool-cancel") {
+        throw new Error("tool cancellation failed");
+      }
+    });
 
-    abortController.abort(abortError);
+    expect(() => abortController.abort(abortError)).not.toThrow();
     await Promise.resolve();
 
     expect(onRejected).toHaveBeenCalledWith(abortError);
@@ -217,17 +343,26 @@ describe("AssistantFrameHost", () => {
       },
       DEFAULT_ORIGIN,
     );
+    expect(consoleError).toHaveBeenCalledOnce();
     expect(vi.getTimerCount()).toBe(0);
     host.dispose();
   });
 
   it("cancels tool calls when they time out", async () => {
     const { execute, getToolCallId, host, postMessage } = createHost();
+    const consoleError = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => undefined);
     const result = Promise.resolve(execute({}, executionContext));
     const toolCallId = getToolCallId();
     const rejection = expect(result).rejects.toThrow(
       'Tool call "search" timed out',
     );
+    postMessage.mockImplementation((data) => {
+      if (data.message.type === "tool-cancel") {
+        throw new Error("tool cancellation failed");
+      }
+    });
 
     await vi.advanceTimersByTimeAsync(30000);
 
@@ -239,6 +374,7 @@ describe("AssistantFrameHost", () => {
       },
       DEFAULT_ORIGIN,
     );
+    expect(consoleError).toHaveBeenCalledOnce();
     expect(vi.getTimerCount()).toBe(0);
     host.dispose();
   });

package/src/model-context/frame/host.ts

@@ -12,6 +12,13 @@ import {
 
 const getDefaultTargetOrigin = () => window.location.origin;
 
+const logCancellationFailure = (error: unknown) => {
+  console.error(
+    "[assistant-ui] AssistantFrameHost tool cancellation could not be sent.",
+    error,
+  );
+};
+
 /**
  * Deserializes tools from JSON Schema format back to Tool objects
  */
@@ -73,9 +80,10 @@ export class AssistantFrameHost implements ModelContextProvider {
     this._targetOrigin = targetOrigin;
 
     this.handleMessage = this.handleMessage.bind(this);
-    window.addEventListener("message", this.handleMessage);
-
+    // A posted message cannot be delivered before this constructor returns, so
+    // requesting first means a failed request leaves no listener to clean up.
     this.requestContext();
+    window.addEventListener("message", this.handleMessage);
   }
 
   private handleMessage(event: MessageEvent) {
@@ -195,21 +203,34 @@ export class AssistantFrameHost implements ModelContextProvider {
       }, timeout);
       abortSignal?.addEventListener("abort", onAbort, { once: true });
 
-      this._iframeWindow.postMessage(
-        { channel: FRAME_MESSAGE_CHANNEL, message },
-        this._targetOrigin,
-      );
+      try {
+        this._iframeWindow.postMessage(
+          { channel: FRAME_MESSAGE_CHANNEL, message },
+          this._targetOrigin,
+        );
+      } catch (error) {
+        const pending = this._pendingRequests.get(message.id);
+        this._pendingRequests.delete(message.id);
+        pending?.reject(error);
+      }
     });
   }
 
-  private cancelToolCall(id: string) {
-    this._iframeWindow.postMessage(
-      {
-        channel: FRAME_MESSAGE_CHANNEL,
-        message: { type: "tool-cancel", id } satisfies FrameMessage,
-      },
-      this._targetOrigin,
-    );
+  private cancelToolCall(
+    id: string,
+    onError: (error: unknown) => void = logCancellationFailure,
+  ) {
+    try {
+      this._iframeWindow.postMessage(
+        {
+          channel: FRAME_MESSAGE_CHANNEL,
+          message: { type: "tool-cancel", id } satisfies FrameMessage,
+        },
+        this._targetOrigin,
+      );
+    } catch (error) {
+      onError(error);
+    }
   }
 
   private requestContext() {
@@ -242,10 +263,22 @@ export class AssistantFrameHost implements ModelContextProvider {
     window.removeEventListener("message", this.handleMessage);
     this._subscribers.clear();
     const error = new Error("AssistantFrameHost has been disposed");
+    let cancellationFailed = false;
+    let cancellationError: unknown;
+
     for (const [id, pending] of this._pendingRequests) {
-      this.cancelToolCall(id);
+      this._pendingRequests.delete(id);
+      this.cancelToolCall(id, (error) => {
+        if (!cancellationFailed) {
+          cancellationFailed = true;
+          cancellationError = error;
+        } else {
+          logCancellationFailure(error);
+        }
+      });
       pending.reject(error);
     }
-    this._pendingRequests.clear();
+
+    if (cancellationFailed) throw cancellationError;
   }
 }