@assistant-ui/core 0.3.16 → 0.3.18

package/src/runtime/base/base-composer-runtime-core.ts

@@ -1,4 +1,5 @@
 import {
+  isAttachmentComplete,
   isCreateAttachment,
   type Attachment,
   type CompleteAttachment,
@@ -34,9 +35,6 @@ import {
   drainAttachmentAdd,
 } from "../utils/attachment-add-operations";
 
-const isAttachmentComplete = (a: Attachment): a is CompleteAttachment =>
-  a.status.type === "complete";
-
 export abstract class BaseComposerRuntimeCore
   extends BaseSubscribable
   implements ComposerRuntimeCore
@@ -161,6 +159,7 @@ export abstract class BaseComposerRuntimeCore
   private _emptyTextAndAttachments() {
     this._attachments = [];
     this._text = "";
+    this._rebaseDictation("");
     this._notifySubscribers();
   }
 
@@ -301,7 +300,10 @@ export abstract class BaseComposerRuntimeCore
       void sendTask.catch((error) => {
         this._restoreUnsentDraft(error, generation, draft);
       });
-    this._notifyEventSubscribers("send", {});
+    this._notifyEventSubscribers("send", {
+      chars: text.length,
+      attachments: finalAttachments.length,
+    });
   }
 
   /**
@@ -417,6 +419,7 @@ export abstract class BaseComposerRuntimeCore
           message,
           undefined,
           err,
+          fileOrAttachment.contentType,
         );
         throw err;
       }
@@ -431,7 +434,9 @@ export abstract class BaseComposerRuntimeCore
       };
       this._attachments = [...this._attachments, a];
       this._notifySubscribers();
-      this._notifyEventSubscribers("attachmentAdd", {});
+      this._notifyEventSubscribers("attachmentAdd", {
+        ...(a.contentType ? { contentType: a.contentType } : undefined),
+      });
       return;
     }
 
@@ -439,7 +444,13 @@ export abstract class BaseComposerRuntimeCore
     if (!adapter) {
       const message = "Attachments are not supported";
       const err = new Error(message);
-      this._safeEmitAttachmentAddError("no-adapter", message, undefined, err);
+      this._safeEmitAttachmentAddError(
+        "no-adapter",
+        message,
+        undefined,
+        err,
+        fileOrAttachment.type,
+      );
       throw err;
     }
 
@@ -451,7 +462,13 @@ export abstract class BaseComposerRuntimeCore
     ) {
       const message = `File type ${fileOrAttachment.type || "unknown"} is not accepted. Accepted types: ${adapter.accept}`;
       const err = new Error(message);
-      this._safeEmitAttachmentAddError("not-accepted", message, undefined, err);
+      this._safeEmitAttachmentAddError(
+        "not-accepted",
+        message,
+        undefined,
+        err,
+        fileOrAttachment.type,
+      );
       throw err;
     }
 
@@ -501,6 +518,7 @@ export abstract class BaseComposerRuntimeCore
         e instanceof Error ? e.message : String(e),
         lastAttachment?.id,
         e instanceof Error ? e : undefined,
+        lastAttachment?.contentType || fileOrAttachment.type,
       );
       throw e;
     } finally {
@@ -517,9 +535,17 @@ export abstract class BaseComposerRuntimeCore
         lastAttachment.status.message ??
           "Attachment upload did not complete successfully.",
         lastAttachment.id,
+        undefined,
+        lastAttachment.contentType || fileOrAttachment.type,
       );
     } else {
-      this._notifyEventSubscribers("attachmentAdd", {});
+      this._notifyEventSubscribers("attachmentAdd", {
+        ...(lastAttachment?.contentType
+          ? { contentType: lastAttachment.contentType }
+          : fileOrAttachment.type
+            ? { contentType: fileOrAttachment.type }
+            : undefined),
+      });
     }
   }
 
@@ -528,6 +554,7 @@ export abstract class BaseComposerRuntimeCore
     message: string,
     attachmentId?: string,
     error?: Error,
+    contentType?: string,
   ) {
     try {
       this._notifyEventSubscribers("attachmentAddError", {
@@ -535,6 +562,7 @@ export abstract class BaseComposerRuntimeCore
         message,
         ...(attachmentId !== undefined && { attachmentId }),
         ...(error !== undefined && { error }),
+        ...(contentType ? { contentType } : undefined),
       });
     } catch (subscriberError) {
       console.error(

package/src/runtime/base/base-thread-runtime-core.test.ts

@@ -1,6 +1,8 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import type { RealtimeVoiceAdapter } from "../../adapters/voice";
+import type { ModelContextProvider } from "../../model-context/types";
 import type { AppendMessage } from "../../types/message";
+import { CompositeContextProvider } from "../../utils/composite-context-provider";
 import type {
   AddToolResultOptions,
   ResumeRunConfig,
@@ -14,6 +16,9 @@ import { BaseThreadRuntimeCore } from "./base-thread-runtime-core";
 
 const createVoiceAdapter = () => {
   let volumeCallback: ((volume: number) => void) | undefined;
+  let transcriptCallback:
+    | ((transcript: RealtimeVoiceAdapter.TranscriptItem) => void)
+    | undefined;
   const session: RealtimeVoiceAdapter.Session = {
     status: { type: "running" },
     isMuted: false,
@@ -21,7 +26,12 @@ const createVoiceAdapter = () => {
     mute: vi.fn(),
     unmute: vi.fn(),
     onStatusChange: () => () => {},
-    onTranscript: () => () => {},
+    onTranscript: (callback) => {
+      transcriptCallback = callback;
+      return () => {
+        transcriptCallback = undefined;
+      };
+    },
     onModeChange: () => () => {},
     onVolumeChange: (callback) => {
       volumeCallback = callback;
@@ -34,10 +44,13 @@ const createVoiceAdapter = () => {
   return {
     adapter: { connect: () => session },
     emitVolume: (volume: number) => volumeCallback?.(volume),
+    emitTranscript: (transcript: RealtimeVoiceAdapter.TranscriptItem) =>
+      transcriptCallback?.(transcript),
     session,
   } satisfies {
     adapter: RealtimeVoiceAdapter;
     emitVolume: (volume: number) => void;
+    emitTranscript: (transcript: RealtimeVoiceAdapter.TranscriptItem) => void;
     session: RealtimeVoiceAdapter.Session;
   };
 };
@@ -45,8 +58,11 @@ const createVoiceAdapter = () => {
 class TestRuntime extends BaseThreadRuntimeCore {
   private readonly voiceAdapter: ReturnType<typeof createVoiceAdapter>;
 
-  constructor(voiceAdapter: ReturnType<typeof createVoiceAdapter>) {
-    super({ getModelContext: () => ({}) });
+  constructor(
+    voiceAdapter: ReturnType<typeof createVoiceAdapter>,
+    contextProvider: ModelContextProvider = { getModelContext: () => ({}) },
+  ) {
+    super(contextProvider);
     this.voiceAdapter = voiceAdapter;
   }
 
@@ -99,7 +115,7 @@ class TestRuntime extends BaseThreadRuntimeCore {
   resumeRun(_config: ResumeRunConfig) {}
   addToolResult(_options: AddToolResultOptions) {}
   resumeToolCall(_options: ResumeToolCallOptions) {}
-  respondToToolApproval(_options: RespondToToolApprovalOptions) {}
+  async respondToToolApproval(_options: RespondToToolApprovalOptions) {}
   cancelRun() {}
   exportExternalState() {
     return {};
@@ -126,6 +142,92 @@ describe("BaseThreadRuntimeCore subscriptions", () => {
     expect(() => runtime.reset()).toThrow(error);
     expect(laterSubscriber).toHaveBeenCalledOnce();
   });
+
+  it("isolates initialize listener errors during late replay", async () => {
+    const consoleError = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => {});
+    const runtime = new TestRuntime(createVoiceAdapter());
+    runtime.reset();
+
+    const syncError = new Error("sync listener failed");
+    const asyncError = new Error("async listener failed");
+    runtime.unstable_on("initialize", () => {
+      throw syncError;
+    });
+    runtime.unstable_on("initialize", async () => {
+      throw asyncError;
+    });
+
+    await vi.waitFor(() => {
+      expect(consoleError).toHaveBeenCalledTimes(2);
+      expect(consoleError).toHaveBeenCalledWith(
+        '[assistant-ui] Thread runtime "initialize" listener threw an error',
+        syncError,
+      );
+      expect(consoleError).toHaveBeenCalledWith(
+        '[assistant-ui] Thread runtime "initialize" listener threw an error',
+        asyncError,
+      );
+    });
+  });
+
+  it("isolates modelContextUpdate listener errors during context updates", () => {
+    const consoleError = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => {});
+    const provider = new CompositeContextProvider();
+    const runtime = new TestRuntime(createVoiceAdapter(), provider);
+    const listenerError = new Error("model context listener failed");
+    const laterListener = vi.fn();
+
+    runtime.unstable_on("modelContextUpdate", () => {
+      throw listenerError;
+    });
+    runtime.unstable_on("modelContextUpdate", laterListener);
+
+    const unregister = provider.registerModelContextProvider({
+      getModelContext: () => ({}),
+    });
+
+    expect(laterListener).toHaveBeenCalledOnce();
+    expect(consoleError).toHaveBeenNthCalledWith(
+      1,
+      '[assistant-ui] Thread runtime "modelContextUpdate" listener threw an error',
+      listenerError,
+    );
+    expect(() => unregister()).not.toThrow();
+    expect(laterListener).toHaveBeenCalledTimes(2);
+    expect(consoleError).toHaveBeenNthCalledWith(
+      2,
+      '[assistant-ui] Thread runtime "modelContextUpdate" listener threw an error',
+      listenerError,
+    );
+  });
+
+  it("observes rejected modelContextUpdate listener promises", async () => {
+    const consoleError = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => {});
+    const provider = new CompositeContextProvider();
+    const runtime = new TestRuntime(createVoiceAdapter(), provider);
+    const listenerError = new Error("async model context listener failed");
+
+    runtime.unstable_on("modelContextUpdate", async () => {
+      throw listenerError;
+    });
+
+    expect(() =>
+      provider.registerModelContextProvider({ getModelContext: () => ({}) }),
+    ).not.toThrow();
+
+    await vi.waitFor(() => {
+      expect(consoleError).toHaveBeenCalledWith(
+        '[assistant-ui] Thread runtime "modelContextUpdate" listener threw an error',
+        listenerError,
+      );
+    });
+  });
 });
 
 describe("BaseThreadRuntimeCore voice volume subscriptions", () => {
@@ -205,3 +307,46 @@ describe("BaseThreadRuntimeCore voice volume subscriptions", () => {
     });
   });
 });
+
+describe("BaseThreadRuntimeCore voice transcripts", () => {
+  it("completes a final-only reply before the next streamed reply", () => {
+    const voiceAdapter = createVoiceAdapter();
+    const runtime = new TestRuntime(voiceAdapter);
+    runtime.connectVoice();
+
+    try {
+      voiceAdapter.emitTranscript({
+        role: "assistant",
+        text: "Finished reply",
+        isFinal: true,
+      });
+      expect(runtime.messages).toMatchObject([
+        {
+          content: [{ type: "text", text: "Finished reply" }],
+          status: { type: "complete", reason: "stop" },
+        },
+      ]);
+
+      voiceAdapter.emitTranscript({ role: "assistant", text: "Next" });
+      expect(runtime.messages.at(-1)?.status).toEqual({ type: "running" });
+
+      voiceAdapter.emitTranscript({
+        role: "assistant",
+        text: "Next reply",
+        isFinal: true,
+      });
+      expect(runtime.messages).toMatchObject([
+        {
+          content: [{ type: "text", text: "Finished reply" }],
+          status: { type: "complete", reason: "stop" },
+        },
+        {
+          content: [{ type: "text", text: "Next reply" }],
+          status: { type: "complete", reason: "stop" },
+        },
+      ]);
+    } finally {
+      runtime.disconnectVoice();
+    }
+  });
+});

package/src/runtime/base/base-thread-runtime-core.ts

@@ -51,7 +51,7 @@ export abstract class BaseThreadRuntimeCore
 {
   private _isInitialized = false;
 
-  protected readonly repository = new MessageRepository();
+  protected repository = new MessageRepository();
   public abstract get adapters(): BaseThreadAdapters | undefined;
   public abstract get isDisabled(): boolean;
   public abstract get isSendDisabled(): boolean;
@@ -68,7 +68,7 @@ export abstract class BaseThreadRuntimeCore
   public abstract resumeToolCall(options: ResumeToolCallOptions): void;
   public abstract respondToToolApproval(
     options: RespondToToolApprovalOptions,
-  ): void;
+  ): Promise<void>;
   public abstract cancelRun(): void;
   public abstract exportExternalState(): any;
   public abstract importExternalState(state: any): void;
@@ -393,6 +393,10 @@ export abstract class BaseThreadRuntimeCore
         this._notifySubscribers();
       }
     } else {
+      const status: ThreadAssistantMessage["status"] = transcript.isFinal
+        ? { type: "complete", reason: "stop" }
+        : { type: "running" };
+
       if (!this._currentAssistantMsg) {
         this._currentAssistantMsg = {
           id: generateId(),
@@ -405,7 +409,7 @@ export abstract class BaseThreadRuntimeCore
             steps: [],
             custom: {},
           },
-          status: { type: "running" },
+          status,
           createdAt: new Date(),
         };
         this._voiceMessages.push(this._currentAssistantMsg);
@@ -415,9 +419,7 @@ export abstract class BaseThreadRuntimeCore
         const updated: ThreadAssistantMessage = {
           ...this._currentAssistantMsg,
           content: [{ type: "text", text: transcript.text }],
-          ...(transcript.isFinal
-            ? { status: { type: "complete", reason: "stop" } }
-            : {}),
+          status,
         };
         this._voiceMessages[idx] = updated;
         this._currentAssistantMsg = updated;
@@ -518,7 +520,11 @@ export abstract class BaseThreadRuntimeCore
     const wrapped = callback as (payload?: unknown) => void;
     if (event === "modelContextUpdate") {
       // provider.subscribe is `() => void`; pump the typed empty payload to the user callback.
-      return this._contextProvider.subscribe?.(() => wrapped({})) ?? (() => {});
+      return (
+        this._contextProvider.subscribe?.(() =>
+          notifyEventListeners([wrapped], {}, `Thread runtime "${event}"`),
+        ) ?? (() => {})
+      );
     }
 
     let subscribers = this._eventSubscribers.get(event);
@@ -532,7 +538,9 @@ export abstract class BaseThreadRuntimeCore
     // after the thread already initialized, mirroring a BehaviorSubject.
     if (event === "initialize" && this._isInitialized) {
       queueMicrotask(() => {
-        if (subscribers.has(wrapped)) wrapped({});
+        if (subscribers.has(wrapped)) {
+          notifyEventListeners([wrapped], {}, `Thread runtime "${event}"`);
+        }
       });
     }
 

package/src/runtime/interfaces/composer-runtime-core.ts

@@ -21,16 +21,21 @@ export type AttachmentAddErrorEvent = {
 
 export type ComposerRuntimeEventPayload = {
   /**
-   * @deprecated State-derivable. Observe `state.text` clearing via
-   * `subscribe` + `getState` instead. Kept for backward compatibility.
+   * Fired after a send with the size of what went out. The composer state is
+   * already cleared when it fires, so the counts are only available here.
    */
-  send: Record<string, never>;
+  send: {
+    readonly chars: number;
+    readonly attachments: number;
+  };
   /**
    * @deprecated State-derivable. Observe `state.attachments` via `subscribe` +
    * `getState` instead. Kept for backward compatibility.
    */
-  attachmentAdd: Record<string, never>;
-  attachmentAddError: AttachmentAddErrorEvent;
+  attachmentAdd: { readonly contentType?: string | undefined };
+  attachmentAddError: AttachmentAddErrorEvent & {
+    readonly contentType?: string | undefined;
+  };
 };
 
 export type ComposerRuntimeEventType = keyof ComposerRuntimeEventPayload;

package/src/runtime/interfaces/thread-list-runtime-core.ts

@@ -1,5 +1,8 @@
 import type { Unsubscribe } from "../../types/unsubscribe";
-import type { ThreadRuntimeCore } from "./thread-runtime-core";
+import type {
+  ThreadRuntimeCore,
+  ThreadRuntimeEventType,
+} from "./thread-runtime-core";
 
 export type ThreadListItemStatus = "archived" | "regular" | "new" | "deleted";
 
@@ -16,8 +19,14 @@ export type ThreadListItemCoreState = {
   readonly runtime?: ThreadRuntimeCore | undefined;
 };
 
+export type ThreadListRuntimeEvent = {
+  readonly threadId: string;
+  readonly type: ThreadRuntimeEventType;
+};
+
 export type ThreadListRuntimeCore = {
   readonly isLoading: boolean;
+  readonly loadError?: unknown;
   readonly isLoadingMore?: boolean;
   readonly hasMore?: boolean;
   mainThreadId: string;
@@ -41,6 +50,17 @@ export type ThreadListRuntimeCore = {
    */
   unstable_isThreadRunning?(threadId: string): boolean;
 
+  /**
+   * Lifecycle events from every thread this list keeps alive, including
+   * threads that are not the main one. Implemented by thread lists that keep
+   * runtimes alive for non-main threads. A thread list that mounts only the
+   * main thread leaves this undefined: its other threads have no runtime and so
+   * emit nothing, and the main thread's runtime is observed directly.
+   */
+  unstable_subscribeThreadEvents?(
+    callback: (event: ThreadListRuntimeEvent) => void,
+  ): Unsubscribe;
+
   getItemById(threadId: string): ThreadListItemCoreState | undefined;
 
   switchToThread(
@@ -67,7 +87,10 @@ export type ThreadListRuntimeCore = {
   initialize(
     threadId: string,
   ): Promise<{ remoteId: string; externalId: string | undefined }>;
-  generateTitle(threadId: string): Promise<void>;
+  generateTitle(
+    threadId: string,
+    options?: { automatic?: boolean },
+  ): Promise<void>;
 
   subscribe(callback: () => void): Unsubscribe;
 };

package/src/runtime/interfaces/thread-runtime-core.ts

@@ -63,6 +63,8 @@ export type RespondToToolApprovalOptions = {
   approved: boolean;
   /** The approval option that produced this decision, when the request carried options. */
   optionId?: string;
+  /** The free-form answer, when the request asked for one. */
+  text?: string;
   reason?: string;
 };
 
@@ -165,7 +167,21 @@ export type ThreadRuntimeCore = Readonly<{
 
   addToolResult: (options: AddToolResultOptions) => void;
   resumeToolCall: (options: ResumeToolCallOptions) => void;
-  respondToToolApproval: (options: RespondToToolApprovalOptions) => void;
+  /**
+   * Records a decision on a tool approval gate. Resolves once the runtime has
+   * accepted the response and rejects when it could not be recorded, so a
+   * caller can leave the gate retryable rather than spending it. A capability
+   * or state precondition still throws synchronously; a failure to record
+   * arrives as a rejection, including one an adapter raises synchronously.
+   *
+   * Acceptance is as far as the runtime can see the response: one that records
+   * the decision locally settles on the record, while one that answers by
+   * resuming a run settles on the resume. A failure of the work the decision
+   * unblocks is reported through the runtime's own error channel, not here.
+   */
+  respondToToolApproval: (
+    options: RespondToToolApprovalOptions,
+  ) => Promise<void>;
 
   speak: (messageId: string) => void;
   stopSpeaking: () => void;

package/src/runtime/utils/auto-status.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "vitest";
+import { getAutoStatus } from "./auto-status";
+
+describe("getAutoStatus", () => {
+  it("reports a cancelled message as incomplete", () => {
+    expect(
+      getAutoStatus(true, false, false, false, undefined, true),
+    ).toMatchObject({ type: "incomplete", reason: "cancelled" });
+  });
+
+  it("keeps a cancelled message incomplete once a later message arrives", () => {
+    expect(
+      getAutoStatus(false, false, false, false, undefined, true),
+    ).toMatchObject({ type: "incomplete", reason: "cancelled" });
+  });
+
+  it("reports an uncancelled message as complete", () => {
+    expect(
+      getAutoStatus(true, false, false, false, undefined, false),
+    ).toMatchObject({ type: "complete", reason: "unknown" });
+  });
+
+  it.each([
+    ["running", { type: "running" }, true, false, false, undefined],
+    [
+      "an interrupted tool call",
+      { type: "requires-action", reason: "interrupt" },
+      false,
+      true,
+      true,
+      undefined,
+    ],
+    [
+      "a pending tool call",
+      { type: "requires-action", reason: "tool-calls" },
+      false,
+      false,
+      true,
+      undefined,
+    ],
+    [
+      "an error",
+      { type: "incomplete", reason: "error", error: "boom" },
+      false,
+      false,
+      false,
+      "boom",
+    ],
+  ])(
+    "keeps %s ahead of cancellation",
+    (_label, expected, isRunning, interrupted, pending, error) => {
+      expect(
+        getAutoStatus(true, isRunning, interrupted, pending, error, true),
+      ).toMatchObject(expected);
+    },
+  );
+});

package/src/runtime/utils/auto-status.ts

@@ -36,6 +36,15 @@ const AUTO_STATUS_COMPLETE = Object.freeze(
     { [symbolAutoStatus]: true },
   ),
 );
+const AUTO_STATUS_CANCELLED = Object.freeze(
+  Object.assign(
+    {
+      type: "incomplete" as const,
+      reason: "cancelled" as const,
+    },
+    { [symbolAutoStatus]: true },
+  ),
+);
 
 const AUTO_STATUS_PENDING = Object.freeze(
   Object.assign(
@@ -66,6 +75,7 @@ export const getAutoStatus = (
   hasInterruptedToolCalls: boolean,
   hasPendingToolCalls: boolean,
   error?: ReadonlyJSONValue,
+  isCancelled?: boolean,
 ): MessageStatus => {
   if (isLast && error) {
     return Object.assign(
@@ -84,7 +94,9 @@ export const getAutoStatus = (
       ? AUTO_STATUS_INTERRUPT
       : hasPendingToolCalls
         ? AUTO_STATUS_PENDING
-        : AUTO_STATUS_COMPLETE;
+        : isCancelled
+          ? AUTO_STATUS_CANCELLED
+          : AUTO_STATUS_COMPLETE;
 };
 
 export const getContentAutoStatus = (