@assistant-ui/core 0.3.18 → 0.3.20

package/src/types/message.ts

@@ -395,6 +395,8 @@ export type MessageTiming = {
   readonly toolCallCount: number;
 };
 
+export type MessageModality = "voice";
+
 export type ThreadStep = {
   readonly messageId?: string;
   readonly usage?:
@@ -420,6 +422,7 @@ export type ThreadSystemMessage = MessageCommonProps & {
     readonly steps?: undefined;
     readonly submittedFeedback?: undefined;
     readonly timing?: undefined;
+    readonly modality?: undefined;
     readonly custom: Record<string, unknown>;
   };
 };
@@ -436,6 +439,8 @@ export type ThreadUserMessage = MessageCommonProps & {
     readonly submittedFeedback?: undefined;
     readonly timing?: undefined;
     readonly isOptimistic?: boolean;
+    /** Set when the message was produced by a voice session rather than typed. */
+    readonly modality?: MessageModality;
     readonly custom: Record<string, unknown>;
   };
 };
@@ -449,13 +454,18 @@ export type ThreadAssistantMessage = MessageCommonProps & {
     readonly unstable_annotations: readonly ReadonlyJSONValue[];
     readonly unstable_data: readonly ReadonlyJSONValue[];
     readonly steps: readonly ThreadStep[];
-    readonly submittedFeedback?: { readonly type: "positive" | "negative" };
+    readonly submittedFeedback?: {
+      readonly type: "positive" | "negative";
+      readonly comment?: string;
+    };
     readonly timing?: MessageTiming;
     /**
      * Marks a client-side optimistic placeholder. Such messages are evicted
      * once off the head branch and are never persisted.
      */
     readonly isOptimistic?: boolean;
+    /** Set when the message was produced by a voice session rather than typed. */
+    readonly modality?: MessageModality;
     readonly custom: Record<string, unknown>;
   };
 };
@@ -463,13 +473,19 @@ export type ThreadAssistantMessage = MessageCommonProps & {
 type BaseThreadMessage = {
   readonly status?: ThreadAssistantMessage["status"];
   readonly metadata: {
-    readonly unstable_state?: ReadonlyJSONValue;
-    readonly unstable_annotations?: readonly ReadonlyJSONValue[];
-    readonly unstable_data?: readonly ReadonlyJSONValue[];
-    readonly steps?: readonly ThreadStep[];
-    readonly submittedFeedback?: { readonly type: "positive" | "negative" };
-    readonly timing?: MessageTiming;
+    readonly unstable_state?: ReadonlyJSONValue | undefined;
+    readonly unstable_annotations?: readonly ReadonlyJSONValue[] | undefined;
+    readonly unstable_data?: readonly ReadonlyJSONValue[] | undefined;
+    readonly steps?: readonly ThreadStep[] | undefined;
+    readonly submittedFeedback?:
+      | {
+          readonly type: "positive" | "negative";
+          readonly comment?: string;
+        }
+      | undefined;
+    readonly timing?: MessageTiming | undefined;
     readonly isOptimistic?: boolean;
+    readonly modality?: MessageModality | undefined;
     readonly custom: Record<string, unknown>;
   };
   readonly attachments?: ThreadUserMessage["attachments"];

package/src/utils/abortable-iterable.test.ts

@@ -16,6 +16,121 @@ const collect = async <T>(iterable: AsyncIterable<T>) => {
 };
 
 describe("abortableIterable", () => {
+  it.each([false, true])(
+    "releases the opening abort listener on rejection=%s",
+    async (fails) => {
+      const controller = new AbortController();
+      const added = vi.spyOn(controller.signal, "addEventListener");
+      const removed = vi.spyOn(controller.signal, "removeEventListener");
+      const source: AsyncIterable<number> = {
+        async *[Symbol.asyncIterator]() {
+          yield 1;
+        },
+      };
+      const opening = openAbortableIterable(
+        fails ? Promise.reject(new Error("opening failed")) : source,
+        controller.signal,
+      );
+      if (fails) await expect(opening).rejects.toThrow("opening failed");
+      else await expect(opening).resolves.toBe(source);
+      expect(added).toHaveBeenCalledTimes(1);
+      expect(removed).toHaveBeenCalledWith("abort", added.mock.calls[0]![1]);
+    },
+  );
+
+  it("finalizes once when an aborted read later rejects", async () => {
+    let rejectRead!: (error: unknown) => void;
+    const read = new Promise<IteratorResult<number>>((_, reject) => {
+      rejectRead = reject;
+    });
+    const finalize = vi.fn(async () => ({
+      done: true as const,
+      value: undefined,
+    }));
+    const source: AsyncIterable<number> = {
+      [Symbol.asyncIterator]: () => ({ next: () => read, return: finalize }),
+    };
+    const controller = new AbortController();
+    const iterator = abortableIterable(source, controller.signal)[
+      Symbol.asyncIterator
+    ]();
+    const pending = iterator.next();
+    controller.abort();
+    await expect(pending).resolves.toEqual({ done: true, value: undefined });
+    expect(finalize).toHaveBeenCalledTimes(1);
+    rejectRead(new Error("late failure"));
+    await read.catch(() => {});
+    await iterator.return?.();
+    expect(finalize).toHaveBeenCalledTimes(1);
+  });
+
+  it("releases the listener and finalizes after a synchronous read failure", async () => {
+    const controller = new AbortController();
+    const added = vi.spyOn(controller.signal, "addEventListener");
+    const removed = vi.spyOn(controller.signal, "removeEventListener");
+    const error = new Error("read failed");
+    const finalize = vi.fn(async () => ({
+      done: true as const,
+      value: undefined,
+    }));
+    const source: AsyncIterable<number> = {
+      [Symbol.asyncIterator]: () => ({
+        next: () => {
+          throw error;
+        },
+        return: finalize,
+      }),
+    };
+    await expect(
+      collect(abortableIterable(source, controller.signal)),
+    ).rejects.toBe(error);
+    expect(removed).toHaveBeenCalledWith("abort", added.mock.calls[0]![1]);
+    expect(finalize).toHaveBeenCalledTimes(1);
+    controller.abort();
+    expect(finalize).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not read or finalize again after the consumer returns", async () => {
+    const next = vi.fn(async () => ({ done: false as const, value: 1 }));
+    const finalize = vi.fn(async () => ({
+      done: true as const,
+      value: undefined,
+    }));
+    const source: AsyncIterable<number> = {
+      [Symbol.asyncIterator]: () => ({ next, return: finalize }),
+    };
+    const iterator = abortableIterable(source, new AbortController().signal)[
+      Symbol.asyncIterator
+    ]();
+    await iterator.return?.();
+    await iterator.return?.();
+    await expect(iterator.next()).resolves.toEqual({
+      done: true,
+      value: undefined,
+    });
+    expect(next).not.toHaveBeenCalled();
+    expect(finalize).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not finalize a naturally exhausted source", async () => {
+    const next = vi.fn(async () => ({ done: true as const, value: undefined }));
+    const finalize = vi.fn(async () => ({
+      done: true as const,
+      value: undefined,
+    }));
+    const source: AsyncIterable<number> = {
+      [Symbol.asyncIterator]: () => ({ next, return: finalize }),
+    };
+    const iterator = abortableIterable(source, new AbortController().signal)[
+      Symbol.asyncIterator
+    ]();
+    await iterator.next();
+    await iterator.next();
+    await iterator.return?.();
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(finalize).not.toHaveBeenCalled();
+  });
+
   it("settles an opening stream on abort and finalizes a late iterable", async () => {
     const opened = deferred<AsyncIterable<number>>();
     const finalize = vi.fn(async () => ({

package/src/utils/abortable-iterable.ts

@@ -1,24 +1,28 @@
 const done = <T>(): IteratorResult<T> => ({ done: true, value: undefined });
 
-/** Resolves at cancellation, so an await on a stream can stop being the only way out. */
-const whenAborted = (signal: AbortSignal): Promise<undefined> =>
-  new Promise((resolve) => {
-    if (signal.aborted) return resolve(undefined);
-    signal.addEventListener("abort", () => resolve(undefined), { once: true });
-  });
-
 export const openAbortableIterable = async <T>(
   source: AsyncIterable<T> | Promise<AsyncIterable<T>>,
   signal: AbortSignal,
 ): Promise<AsyncIterable<T> | undefined> => {
   const opened = Promise.resolve(source);
-  const iterable = await Promise.race([opened, whenAborted(signal)]);
-  if (!iterable) {
-    void opened
-      .then((late) => late[Symbol.asyncIterator]().return?.(undefined))
-      .catch(() => {});
+  let release = () => {};
+  const aborted = new Promise<undefined>((resolve) => {
+    const onAbort = () => resolve(undefined);
+    if (signal.aborted) return onAbort();
+    signal.addEventListener("abort", onAbort, { once: true });
+    release = () => signal.removeEventListener("abort", onAbort);
+  });
+  try {
+    const iterable = await Promise.race([opened, aborted]);
+    if (!iterable) {
+      void opened
+        .then((late) => late[Symbol.asyncIterator]().return?.(undefined))
+        .catch(() => {});
+    }
+    return iterable;
+  } finally {
+    release();
   }
-  return iterable;
 };
 
 /**
@@ -40,7 +44,10 @@ export const abortableIterable = <T>(
 ): AsyncIterable<T> => ({
   [Symbol.asyncIterator]: () => {
     const iterator = source[Symbol.asyncIterator]();
+    let finished = false;
     const finalize = () => {
+      if (finished) return;
+      finished = true;
       try {
         iterator.return?.(undefined)?.catch(() => {});
       } catch {
@@ -53,7 +60,7 @@ export const abortableIterable = <T>(
         return this;
       },
       next: () => {
-        if (signal.aborted) {
+        if (finished || signal.aborted) {
           finalize();
           return Promise.resolve(done<T>());
         }
@@ -68,17 +75,21 @@ export const abortableIterable = <T>(
           signal.addEventListener("abort", onAbort, { once: true });
           const release = () => signal.removeEventListener("abort", onAbort);
 
-          iterator.next().then(
-            (result) => {
+          const onError = (error: unknown) => {
+            release();
+            finalize();
+            reject(error);
+          };
+          try {
+            void iterator.next().then((result) => {
               release();
+              if (finished) return resolve(done<T>());
+              if (result.done) finished = true;
               resolve(result);
-            },
-            (error: unknown) => {
-              release();
-              finalize();
-              reject(error);
-            },
-          );
+            }, onError);
+          } catch (error) {
+            onError(error);
+          }
         });
       },
       // The consumer breaks out of the loop to stop a cancelled run, so this is

package/src/utils/getGroupStatus.test.ts

@@ -1,5 +1,13 @@
 import { describe, expect, it } from "vitest";
-import { getGroupStatus } from "./getGroupStatus";
+import type {
+  MessagePartStatus,
+  ToolCallMessagePartStatus,
+} from "../types/message";
+import { getGroupStatus, getGroupSummary } from "./getGroupStatus";
+
+type Status = MessagePartStatus | ToolCallMessagePartStatus;
+
+const part = (status: Status) => ({ status });
 
 describe("getGroupStatus", () => {
   it("reports running when the first member is running and the last is complete", () => {
@@ -10,7 +18,6 @@ describe("getGroupStatus", () => {
     const status = getGroupStatus(parts);
 
     expect(status).toEqual({ type: "running" });
-    expect(getGroupStatus(parts, [0, 1])).toBe(status);
     expect(status).toBe(getGroupStatus([{ status: { type: "running" } }]));
     expect(Object.isFrozen(status)).toBe(true);
   });
@@ -27,10 +34,69 @@ describe("getGroupStatus", () => {
 
   it("matches the positional outcome for statusless adapters", () => {
     expect(
-      getGroupStatus(
+      getGroupSummary(
         [{ status: { type: "complete" } }, { status: { type: "running" } }],
         [0, 1],
-      ),
+      ).status,
     ).toEqual({ type: "running" });
   });
 });
+
+describe("getGroupSummary", () => {
+  it("tallies every status over the requested indices and reports running when any part runs", () => {
+    const summary = getGroupSummary(
+      [
+        part({ type: "complete" }),
+        part({ type: "running" }),
+        part({ type: "incomplete", reason: "error" }),
+        part({ type: "requires-action", reason: "tool-calls" }),
+      ],
+      [0, 1, 2, 3],
+    );
+
+    expect(summary.counts).toEqual({
+      running: 1,
+      complete: 1,
+      incomplete: 1,
+      requiresAction: 1,
+    });
+    expect(
+      Object.values(summary.counts).reduce((sum, count) => sum + count),
+    ).toBe(4);
+    expect(summary.status).toEqual({ type: "running" });
+  });
+
+  it("uses the last status when no part runs and treats missing parts as complete", () => {
+    const summary = getGroupSummary(
+      [
+        undefined,
+        part({ type: "incomplete", reason: "cancelled" }),
+        part({ type: "requires-action", reason: "interrupt" }),
+      ],
+      [0, 1, 2],
+    );
+
+    expect(summary.counts).toEqual({
+      running: 0,
+      complete: 1,
+      incomplete: 1,
+      requiresAction: 1,
+    });
+    expect(summary.status).toEqual({
+      type: "requires-action",
+      reason: "interrupt",
+    });
+  });
+
+  it("returns complete with zero counts for empty indices", () => {
+    expect(getGroupSummary([], [])).toEqual({
+      status: { type: "complete" },
+      counts: {
+        running: 0,
+        complete: 0,
+        incomplete: 0,
+        requiresAction: 0,
+      },
+    });
+  });
+});

package/src/utils/getGroupStatus.ts

@@ -10,22 +10,45 @@ type PartWithStatus = {
 
 export const getGroupStatus = (
   parts: readonly (PartWithStatus | undefined)[],
-  indices?: readonly number[],
 ): MessagePartStatus | ToolCallMessagePartStatus => {
-  if (indices) {
-    for (const index of indices) {
-      if (parts[index]?.status.type === "running") return RUNNING_STATUS;
-    }
-
-    const lastIndex = indices.at(-1);
-    return lastIndex === undefined
-      ? COMPLETE_STATUS
-      : (parts[lastIndex]?.status ?? COMPLETE_STATUS);
-  }
-
   for (const part of parts) {
     if (part?.status.type === "running") return RUNNING_STATUS;
   }
 
   return parts.at(-1)?.status ?? COMPLETE_STATUS;
 };
+
+export const getGroupSummary = (
+  parts: readonly (PartWithStatus | undefined)[],
+  indices: readonly number[],
+) => {
+  const counts = {
+    running: 0,
+    complete: 0,
+    incomplete: 0,
+    requiresAction: 0,
+  };
+  let status: MessagePartStatus | ToolCallMessagePartStatus = COMPLETE_STATUS;
+  let isRunning = false;
+
+  for (const index of indices) {
+    status = parts[index]?.status ?? COMPLETE_STATUS;
+    switch (status.type) {
+      case "running":
+        counts.running++;
+        isRunning = true;
+        break;
+      case "complete":
+        counts.complete++;
+        break;
+      case "incomplete":
+        counts.incomplete++;
+        break;
+      case "requires-action":
+        counts.requiresAction++;
+        break;
+    }
+  }
+
+  return { status: isRunning ? RUNNING_STATUS : status, counts };
+};

package/src/utils/wire-media.test.ts

@@ -20,6 +20,16 @@ describe("resolveImageMediaType", () => {
     );
   });
 
+  it("ignores a wildcard image content type", () => {
+    expect(resolveImageMediaType(JPEG, "image/*")).toBe("image/jpeg");
+  });
+
+  it("ignores a wildcard data url media type", () => {
+    expect(resolveImageMediaType(`data:image/*;base64,${JPEG}`)).toBe(
+      "image/jpeg",
+    );
+  });
+
   it("takes the data url declaration when it is an image type", () => {
     expect(
       resolveImageMediaType("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"),

package/src/utils/wire-media.ts

@@ -14,10 +14,14 @@ export function resolveImageMediaType(
   image: string,
   contentType?: string | undefined,
 ): string {
-  if (contentType?.startsWith("image/")) return contentType;
+  if (contentType?.startsWith("image/") && !contentType.includes("*")) {
+    return contentType;
+  }
 
   const declared = dataUrlMediaType(image);
-  if (declared?.startsWith("image/")) return declared;
+  if (declared?.startsWith("image/") && !declared.includes("*")) {
+    return declared;
+  }
 
   // Read through a data URL envelope too, so a generic one such as
   // `application/octet-stream` does not mask the format. A url of any other