@assistant-ui/react 0.15.28 → 0.15.29

package/src/adapters/OpenAIRealtimeVoiceAdapter.ts

@@ -0,0 +1,388 @@
+import {
+  createVoiceSession,
+  type RealtimeVoiceAdapter,
+} from "@assistant-ui/core";
+
+/** Options for a browser WebRTC connection to OpenAI Realtime. */
+export type OpenAIRealtimeVoiceAdapterOptions = {
+  /** Exchange the browser's SDP offer for an answer through your authenticated server. Keep the API key and session configuration on the server. */
+  exchangeSdp: (
+    sdp: string,
+    options: { signal: AbortSignal },
+  ) => Promise<string>;
+};
+
+const asRecord = (value: unknown): Record<string, unknown> | undefined =>
+  value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+
+/**
+ * Connects browser microphone audio directly to OpenAI Realtime over WebRTC.
+ * Configure audio output, input transcription, and automatic turn detection on
+ * the server. Each connection starts a new conversation; existing thread history
+ * and assistant-ui tools are not sent to the voice model.
+ */
+export class OpenAIRealtimeVoiceAdapter implements RealtimeVoiceAdapter {
+  private readonly options: OpenAIRealtimeVoiceAdapterOptions;
+
+  constructor(options: OpenAIRealtimeVoiceAdapterOptions) {
+    this.options = options;
+  }
+
+  connect(options: {
+    abortSignal?: AbortSignal;
+  }): RealtimeVoiceAdapter.Session {
+    return createVoiceSession(options, async (session) => {
+      if (
+        typeof RTCPeerConnection === "undefined" ||
+        typeof document === "undefined" ||
+        typeof navigator === "undefined" ||
+        !navigator.mediaDevices?.getUserMedia
+      ) {
+        throw new Error(
+          "OpenAI Realtime voice requires a browser with WebRTC and microphone access.",
+        );
+      }
+
+      const controller = new AbortController();
+      const peer = new RTCPeerConnection();
+      const channel = peer.createDataChannel("oai-events");
+      const audio = document.createElement("audio");
+      audio.autoplay = true;
+      let microphone: MediaStream | undefined;
+      let muted = false;
+      let closed = false;
+      let generating = false;
+      let speaking = false;
+      let transcribing = false;
+      let volumeTimer: ReturnType<typeof setInterval> | undefined;
+      let connectionTimer: ReturnType<typeof setTimeout> | undefined;
+      type Transcript = {
+        ready: boolean;
+        value: RealtimeVoiceAdapter.TranscriptItem | undefined;
+      };
+      let assistantTranscript: (Transcript & { id: string }) | undefined;
+      const transcriptQueue: Transcript[] = [];
+      const pendingTranscripts = new Map<string, Transcript>();
+      const flushTranscripts = () => {
+        while (transcriptQueue[0]?.ready) {
+          const item = transcriptQueue[0];
+          item.ready = false;
+          if (item.value?.isFinal !== false) transcriptQueue.shift();
+          if (item.value) session.emitTranscript(item.value);
+        }
+      };
+      const finishUserTranscript = (
+        id: unknown,
+        value?: RealtimeVoiceAdapter.TranscriptItem,
+      ) => {
+        const item =
+          typeof id === "string" ? pendingTranscripts.get(id) : undefined;
+        if (item) {
+          item.ready = true;
+          item.value = value;
+          pendingTranscripts.delete(id as string);
+        } else if (value) transcriptQueue.push({ ready: true, value });
+        flushTranscripts();
+      };
+      const pending = new Map<
+        string,
+        {
+          resolve: () => void;
+          reject: (error: Error) => void;
+          timer: ReturnType<typeof setTimeout>;
+        }
+      >();
+
+      const disconnect = () => {
+        if (closed) return;
+        closed = true;
+        controller.abort();
+        clearInterval(volumeTimer);
+        clearTimeout(connectionTimer);
+        for (const id of pending.keys())
+          settle(
+            id,
+            new Error(
+              "OpenAI Realtime session disconnected before accepting the message.",
+            ),
+          );
+        pendingTranscripts.clear();
+        transcriptQueue.length = 0;
+        channel.onopen =
+          channel.onmessage =
+          channel.onclose =
+          channel.onerror =
+            null;
+        peer.ontrack = peer.onconnectionstatechange = null;
+        for (const track of microphone?.getTracks() ?? []) track.stop();
+        audio.pause();
+        audio.srcObject = null;
+        channel.close();
+        peer.close();
+      };
+
+      const end = (reason: "finished" | "error", error?: unknown) => {
+        session.end(reason, error);
+        disconnect();
+      };
+
+      const send = (event: Record<string, unknown>) => {
+        if (closed || channel.readyState !== "open")
+          throw new Error("OpenAI Realtime session is not connected.");
+        channel.send(JSON.stringify(event));
+      };
+
+      const request = (id: string, event: Record<string, unknown>) =>
+        new Promise<void>((resolve, reject) => {
+          const timer = setTimeout(() => {
+            reject(
+              new Error("OpenAI Realtime did not acknowledge the message."),
+            );
+          }, 10_000);
+          pending.set(id, { resolve, reject, timer });
+          send({ ...event, event_id: id });
+        }).finally(() => settle(id));
+
+      const settle = (id: unknown, error?: Error) => {
+        if (typeof id !== "string") return false;
+        const message = pending.get(id);
+        if (!message) return false;
+        clearTimeout(message.timer);
+        pending.delete(id);
+        if (error) message.reject(error);
+        else message.resolve();
+        return true;
+      };
+
+      const setSpeaking = (value: boolean) => {
+        speaking = value;
+        session.emitMode(value ? "speaking" : "listening");
+      };
+
+      channel.onmessage = ({ data }) => {
+        if (closed) return;
+        let event: Record<string, unknown> | undefined;
+        try {
+          event = asRecord(JSON.parse(data));
+        } catch {
+          return;
+        }
+        if (!event) return;
+        switch (event.type) {
+          case "session.created":
+          case "session.updated": {
+            const audio = asRecord(asRecord(event.session)?.audio);
+            const input = asRecord(audio?.input);
+            transcribing = asRecord(input?.transcription) !== undefined;
+            break;
+          }
+          case "input_audio_buffer.committed":
+            if (
+              transcribing &&
+              typeof event.item_id === "string" &&
+              !pendingTranscripts.has(event.item_id)
+            ) {
+              const item = { ready: false, value: undefined };
+              pendingTranscripts.set(event.item_id, item);
+              transcriptQueue.push(item);
+            }
+            break;
+          case "conversation.item.input_audio_transcription.completed":
+            if (typeof event.transcript === "string")
+              finishUserTranscript(event.item_id, {
+                role: "user",
+                text: event.transcript,
+                isFinal: true,
+              });
+            break;
+          case "response.output_audio_transcript.delta":
+          case "response.output_audio_transcript.done": {
+            if (typeof event.item_id !== "string") break;
+            const final =
+              event.type === "response.output_audio_transcript.done";
+            const text = final ? event.transcript : event.delta;
+            if (typeof text !== "string") break;
+            if (assistantTranscript?.id !== event.item_id) {
+              assistantTranscript = {
+                id: event.item_id,
+                ready: false,
+                value: undefined,
+              };
+              transcriptQueue.push(assistantTranscript);
+            }
+            assistantTranscript.ready = true;
+            assistantTranscript.value = {
+              role: "assistant",
+              text: final
+                ? text
+                : (assistantTranscript.value?.text ?? "") + text,
+              isFinal: final,
+            };
+            flushTranscripts();
+            break;
+          }
+          case "response.created":
+            generating = true;
+            settle(
+              asRecord(asRecord(event.response)?.metadata)?.aui_request_id,
+            );
+            break;
+          case "response.done": {
+            generating = false;
+            const response = asRecord(event.response);
+            if (assistantTranscript?.value?.isFinal === false) {
+              assistantTranscript.value = {
+                ...assistantTranscript.value,
+                isFinal: true,
+              };
+              assistantTranscript.ready = true;
+              flushTranscripts();
+            }
+            if (response?.status === "failed")
+              console.warn(
+                "OpenAI Realtime response failed.",
+                response.status_details,
+              );
+            break;
+          }
+          case "output_audio_buffer.started":
+            setSpeaking(true);
+            break;
+          case "output_audio_buffer.stopped":
+          case "output_audio_buffer.cleared":
+            setSpeaking(false);
+            break;
+          case "conversation.item.added":
+            settle(asRecord(event.item)?.id);
+            break;
+          case "conversation.item.input_audio_transcription.failed":
+            finishUserTranscript(event.item_id);
+            console.warn("OpenAI Realtime transcription failed.", event.error);
+            break;
+          case "error": {
+            const error = asRecord(event.error);
+            if (error?.code === "response_cancel_not_active") break;
+            if (
+              !settle(
+                error?.event_id,
+                new Error(
+                  typeof error?.message === "string"
+                    ? error.message
+                    : "OpenAI Realtime rejected the message.",
+                ),
+              )
+            )
+              console.warn("OpenAI Realtime error.", event.error);
+            break;
+          }
+        }
+      };
+      channel.onopen = () => {
+        if (closed) return;
+        clearTimeout(connectionTimer);
+        session.setStatus({ type: "running" });
+        if (closed) return;
+        session.emitMode("listening");
+        if (closed) return;
+        volumeTimer = setInterval(() => {
+          void peer
+            .getStats()
+            .then((report) => {
+              if (closed) return;
+              let volume = 0;
+              report.forEach((stat) => {
+                if (
+                  (stat.type === "inbound-rtp" ||
+                    (!muted && stat.type === "media-source")) &&
+                  typeof stat.audioLevel === "number"
+                )
+                  volume = Math.max(volume, stat.audioLevel);
+              });
+              session.emitVolume(Math.min(1, volume));
+            })
+            .catch(() => {});
+        }, 100);
+      };
+      channel.onclose = () => end("finished");
+      channel.onerror = () =>
+        end("error", new Error("OpenAI Realtime data channel failed."));
+      peer.onconnectionstatechange = () => {
+        if (peer.connectionState === "failed") {
+          end("error", new Error("OpenAI Realtime connection failed."));
+        } else if (peer.connectionState === "closed") {
+          end("finished");
+        }
+      };
+      peer.ontrack = ({ track }) => {
+        if (closed || track.kind !== "audio") return;
+        audio.srcObject = new MediaStream([track]);
+        void audio.play().catch((error: unknown) => end("error", error));
+      };
+
+      const start = async () => {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          audio: true,
+        });
+        if (closed || session.isDisposed()) {
+          for (const track of stream.getTracks()) track.stop();
+          return;
+        }
+        microphone = stream;
+        for (const track of stream.getAudioTracks()) {
+          track.enabled = !muted;
+          peer.addTrack(track, stream);
+        }
+        connectionTimer = setTimeout(
+          () =>
+            end("error", new Error("OpenAI Realtime connection timed out.")),
+          30_000,
+        );
+        const offer = await peer.createOffer();
+        if (closed) return;
+        await peer.setLocalDescription(offer);
+        if (closed) return;
+        const answer = await this.options.exchangeSdp(offer.sdp!, {
+          signal: controller.signal,
+        });
+        if (closed) return;
+        await peer.setRemoteDescription({ type: "answer", sdp: answer });
+      };
+      void start().catch((error: unknown) => end("error", error));
+
+      return {
+        disconnect,
+        mute: () => {
+          muted = true;
+          for (const track of microphone?.getAudioTracks() ?? [])
+            track.enabled = false;
+        },
+        unmute: () => {
+          muted = false;
+          for (const track of microphone?.getAudioTracks() ?? [])
+            track.enabled = true;
+        },
+        sendText: async (text) => {
+          if (generating) send({ type: "response.cancel" });
+          if (speaking) send({ type: "output_audio_buffer.clear" });
+          const id = crypto.randomUUID().replaceAll("-", "");
+          await request(id, {
+            type: "conversation.item.create",
+            item: {
+              id,
+              type: "message",
+              role: "user",
+              content: [{ type: "input_text", text }],
+            },
+          });
+          const responseId = crypto.randomUUID().replaceAll("-", "");
+          await request(responseId, {
+            type: "response.create",
+            response: { metadata: { aui_request_id: responseId } },
+          });
+        },
+      };
+    });
+  }
+}

package/src/index.ts

@@ -65,6 +65,10 @@ export { CloudFileAttachmentAdapter } from "./legacy-runtime/runtime-cores/adapt
 // --- adapters/voice ---
 export type { RealtimeVoiceAdapter } from "@assistant-ui/core";
 export { createVoiceSession } from "@assistant-ui/core";
+export {
+  OpenAIRealtimeVoiceAdapter,
+  type OpenAIRealtimeVoiceAdapterOptions,
+} from "./adapters/OpenAIRealtimeVoiceAdapter";
 export type {
   VoiceSessionControls,
   VoiceSessionHelpers,

package/src/mcp-apps/McpAppRenderer.test.tsx

@@ -725,6 +725,71 @@ describe("McpAppRenderer", () => {
     expect(appendMock).toHaveBeenCalledTimes(2);
   });
 
+  it("extracts sparse message content without traversing holes", async () => {
+    render(<Harness host={loadingHost()} />);
+    await waitFor(() => expect(framePropsMock).toHaveBeenCalled());
+    const handlers = framePropsMock.mock.lastCall?.[0]
+      .handlers as McpAppBridgeHandlers;
+    const content: unknown[] = new Array(50_000_000);
+    content[0] = { type: "text", text: "sparse prompt" };
+
+    const lastIndex = String(content.length - 1);
+    const arrayPrototypeLength = Array.prototype.length;
+    const visitedHole = vi.fn();
+    Object.defineProperty(Array.prototype, lastIndex, {
+      configurable: true,
+      get() {
+        visitedHole();
+        return undefined;
+      },
+    });
+
+    try {
+      const clonedContent = structuredClone(content);
+      expect(
+        handlers.sendMessage?.({ role: "user", content: clonedContent }),
+      ).toEqual({ ok: true });
+    } finally {
+      delete (Array.prototype as unknown[])[Number(lastIndex)];
+      Array.prototype.length = arrayPrototypeLength;
+    }
+
+    expect(visitedHole).not.toHaveBeenCalled();
+    expect(appendMock).toHaveBeenCalledWith({
+      content: [{ type: "text", text: "sparse prompt" }],
+    });
+  });
+
+  it("ignores non-index message content properties after structured clone", async () => {
+    render(<Harness host={loadingHost()} />);
+    await waitFor(() => expect(framePropsMock).toHaveBeenCalled());
+    const handlers = framePropsMock.mock.lastCall?.[0]
+      .handlers as McpAppBridgeHandlers;
+
+    for (const property of ["filter", "constructor"] as const) {
+      const content: unknown[] = [{ type: "text", text: "message text" }];
+      Object.defineProperty(content, property, {
+        configurable: true,
+        enumerable: true,
+        value: { type: "text", text: "not an array element" },
+      });
+      const clonedContent = structuredClone(content);
+
+      expect(Object.hasOwn(clonedContent, property)).toBe(true);
+      expect(
+        handlers.sendMessage?.({ role: "user", content: clonedContent }),
+      ).toEqual({ ok: true });
+    }
+
+    expect(appendMock).toHaveBeenCalledTimes(2);
+    expect(appendMock).toHaveBeenNthCalledWith(1, {
+      content: [{ type: "text", text: "message text" }],
+    });
+    expect(appendMock).toHaveBeenNthCalledWith(2, {
+      content: [{ type: "text", text: "message text" }],
+    });
+  });
+
   it("uses caller UI handlers and keeps data-plane handlers on the host", async () => {
     const host: McpAppsHost = {
       loadResource: vi.fn(async ({ uri }) => ({

package/src/mcp-apps/McpAppRenderer.tsx

@@ -142,8 +142,19 @@ function extractSendMessageTexts(params: unknown): string[] | undefined {
   if (typeof obj["prompt"] === "string") return [obj["prompt"]];
   if (typeof obj["text"] === "string") return [obj["text"]];
   if (typeof obj["message"] === "string") return [obj["message"]];
-  if (Array.isArray(obj["content"])) {
-    return obj["content"]
+  const content = obj["content"];
+  if (Array.isArray(content)) {
+    return Object.keys(content)
+      .filter((key) => {
+        const index = Number(key);
+        return (
+          Number.isInteger(index) &&
+          index >= 0 &&
+          index < content.length &&
+          String(index) === key
+        );
+      })
+      .map((key) => content[Number(key)])
       .filter(
         (block): block is { type: "text"; text: string } =>
           isRecord(block) &&

package/src/model-context/frame/useAssistantFrameHost.detach.test.tsx

@@ -0,0 +1,145 @@
+// @vitest-environment jsdom
+
+import { renderHook } from "@testing-library/react";
+import { AssistantFrameHost, ModelContextRegistry } from "@assistant-ui/core";
+import { expect, it, vi } from "vitest";
+import { useAssistantFrameHost } from "./useAssistantFrameHost";
+
+const attachFrame = () => {
+  const iframe = document.createElement("iframe");
+  document.body.append(iframe);
+  vi.spyOn(iframe.contentWindow!, "postMessage").mockImplementation(() => {});
+  return iframe;
+};
+
+const publishTool = (iframe: HTMLIFrameElement, name: string) => {
+  window.dispatchEvent(
+    new MessageEvent("message", {
+      source: iframe.contentWindow,
+      origin: window.location.origin,
+      data: {
+        channel: "assistant-ui-frame",
+        message: {
+          type: "model-context-update",
+          context: {
+            tools: {
+              [name]: { parameters: { type: "object", properties: {} } },
+            },
+          },
+        },
+      },
+    }),
+  );
+};
+
+it("withdraws a detached frame's tools and calls, then serves an attached frame", async () => {
+  const registry = new ModelContextRegistry();
+  const firstFrame = attachFrame();
+  const iframeRef = { current: firstFrame };
+  const register = (host: Parameters<typeof registry.addProvider>[0]) =>
+    registry.addProvider(host).remove;
+  const { unmount } = renderHook(() =>
+    useAssistantFrameHost({ iframeRef, register }),
+  );
+
+  try {
+    publishTool(firstFrame, "search");
+    const search = registry.getModelContext().tools?.search;
+    expect(search).toBeDefined();
+    const pending = search!.execute!({}, {} as any);
+    const outcome = pending.then(
+      () => undefined,
+      (error: unknown) => error,
+    );
+
+    firstFrame.remove();
+    await vi.waitFor(() => {
+      expect(registry.getModelContext().tools).toBeUndefined();
+    });
+    expect(await outcome).toMatchObject({
+      message: "AssistantFrameHost has been disposed",
+    });
+
+    const secondFrame = attachFrame();
+    iframeRef.current = secondFrame;
+    await vi.waitFor(() => {
+      publishTool(secondFrame, "replacement");
+      expect(registry.getModelContext().tools?.replacement).toBeDefined();
+    });
+    secondFrame.remove();
+  } finally {
+    unmount();
+    firstFrame.remove();
+  }
+});
+
+it("serves a replacement frame before reporting a disposal failure", () => {
+  const disposalError = new Error("tool cancellation failed");
+  const dispose = vi
+    .spyOn(AssistantFrameHost.prototype, "dispose")
+    .mockImplementationOnce(() => {
+      throw disposalError;
+    });
+  const mutationCallbacks: MutationCallback[] = [];
+  vi.stubGlobal(
+    "MutationObserver",
+    class {
+      constructor(callback: MutationCallback) {
+        mutationCallbacks.push(callback);
+      }
+      observe() {}
+      disconnect() {}
+    },
+  );
+
+  const registry = new ModelContextRegistry();
+  const firstFrame = attachFrame();
+  const secondFrame = attachFrame();
+  const iframeRef = { current: firstFrame };
+  const register = (host: Parameters<typeof registry.addProvider>[0]) =>
+    registry.addProvider(host).remove;
+  const { unmount } = renderHook(() =>
+    useAssistantFrameHost({ iframeRef, register }),
+  );
+
+  try {
+    publishTool(firstFrame, "search");
+    expect(registry.getModelContext().tools?.search).toBeDefined();
+
+    iframeRef.current = secondFrame;
+    expect(() => mutationCallbacks[0]!([], {} as MutationObserver)).toThrow(
+      disposalError,
+    );
+
+    expect(registry.getModelContext().tools?.search).toBeUndefined();
+    publishTool(secondFrame, "replacement");
+    expect(registry.getModelContext().tools?.replacement).toBeDefined();
+  } finally {
+    unmount();
+    firstFrame.remove();
+    secondFrame.remove();
+    dispose.mockRestore();
+    vi.unstubAllGlobals();
+  }
+});
+
+it("disposes a host whose registration throws", () => {
+  const dispose = vi.spyOn(AssistantFrameHost.prototype, "dispose");
+  const iframe = attachFrame();
+  const register = vi.fn((): (() => void) => {
+    throw new Error("register failed");
+  });
+
+  try {
+    expect(() =>
+      renderHook(() =>
+        useAssistantFrameHost({ iframeRef: { current: iframe }, register }),
+      ),
+    ).toThrow("register failed");
+    expect(register).toHaveBeenCalledOnce();
+    expect(dispose).toHaveBeenCalledOnce();
+  } finally {
+    dispose.mockRestore();
+    iframe.remove();
+  }
+});