@assistant-ui/react 0.15.31 → 0.15.33

package/src/adapters/gemini-live/audio.ts

@@ -0,0 +1,333 @@
+import {
+  bytesToBase64,
+  raceWithAbortSignal,
+} from "@assistant-ui/core/internal";
+
+export const encodePcm16 = (samples: Float32Array): string => {
+  const bytes = new Uint8Array(samples.length * 2);
+  const view = new DataView(bytes.buffer);
+  for (let i = 0; i < samples.length; i++) {
+    const s = Math.max(-1, Math.min(1, samples[i]!));
+    view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+  }
+  return bytesToBase64(bytes);
+};
+
+export const decodePcm16 = (base64: string): Float32Array<ArrayBuffer> => {
+  const binary = atob(base64);
+  const samples = new Float32Array(new ArrayBuffer((binary.length >> 1) * 4));
+  const view = new DataView(new ArrayBuffer(samples.length * 2));
+  for (let i = 0; i < samples.length * 2; i++)
+    view.setUint8(i, binary.charCodeAt(i));
+  for (let i = 0; i < samples.length; i++)
+    samples[i] = view.getInt16(i * 2, true) / 0x8000;
+  return samples;
+};
+
+export const readLevel = (analyser: AnalyserNode): number => {
+  const data = new Uint8Array(analyser.fftSize);
+  analyser.getByteTimeDomainData(data);
+  let sum = 0;
+  for (const v of data) {
+    const centered = (v - 128) / 128;
+    sum += centered * centered;
+  }
+  return Math.min(1, Math.sqrt(sum / data.length) * 2);
+};
+
+export type MicrophoneCapture = {
+  setMuted: (muted: boolean) => void;
+  level: () => number;
+  close: () => Promise<void>;
+};
+
+const createLowPass = (ratio: number) => {
+  const omega = (Math.PI * 0.9) / ratio;
+  const cosine = Math.cos(omega);
+  const sine = Math.sin(omega);
+  const sections = [0.5176380902, 0.7071067812, 1.9318516526].map((q) => {
+    const alpha = sine / (2 * q);
+    const a0 = 1 + alpha;
+    const b = (1 - cosine) / (2 * a0);
+    return { b, a1: (-2 * cosine) / a0, a2: (1 - alpha) / a0, z1: 0, z2: 0 };
+  });
+  return (samples: Float32Array) => {
+    for (let i = 0; i < samples.length; i++) {
+      let x = samples[i]!;
+      for (const s of sections) {
+        const y = s.b * x + s.z1;
+        s.z1 = 2 * s.b * x - s.a1 * y + s.z2;
+        s.z2 = s.b * x - s.a2 * y;
+        x = y;
+      }
+      samples[i] = x;
+    }
+  };
+};
+
+export const WORKLET_SOURCE = `
+class PcmCapture extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.ratio = sampleRate / options.processorOptions.targetRate;
+    this.chunk = Math.round(options.processorOptions.targetRate / 50);
+    this.position = 0;
+    this.last = 0;
+    this.pending = [];
+    this.sections = [];
+    if (this.ratio > 1) {
+      const omega = (Math.PI * 0.9) / this.ratio;
+      const cosine = Math.cos(omega);
+      const sine = Math.sin(omega);
+      for (const q of [0.5176380902, 0.7071067812, 1.9318516526]) {
+        const alpha = sine / (2 * q);
+        const a0 = 1 + alpha;
+        const b = (1 - cosine) / (2 * a0);
+        this.sections.push({ b, a1: (-2 * cosine) / a0, a2: (1 - alpha) / a0, z1: 0, z2: 0 });
+      }
+    }
+  }
+  process(inputs) {
+    const channels = inputs[0];
+    if (!channels || channels.length === 0 || channels[0].length === 0) return true;
+    const input = channels[0].slice();
+    for (let c = 1; c < channels.length; c++)
+      for (let i = 0; i < input.length; i++) input[i] += channels[c][i];
+    if (channels.length > 1)
+      for (let i = 0; i < input.length; i++) input[i] /= channels.length;
+    for (let i = 0; i < input.length; i++) {
+      let x = input[i];
+      for (const s of this.sections) {
+        const y = s.b * x + s.z1;
+        s.z1 = 2 * s.b * x - s.a1 * y + s.z2;
+        s.z2 = s.b * x - s.a2 * y;
+        x = y;
+      }
+      input[i] = x;
+    }
+    while (this.position < input.length - 1) {
+      const index = Math.floor(this.position);
+      const prev = index < 0 ? this.last : input[index];
+      const next = input[index + 1];
+      this.pending.push(prev + (next - prev) * (this.position - index));
+      this.position += this.ratio;
+    }
+    this.position -= input.length;
+    this.last = input[input.length - 1];
+    if (this.pending.length >= this.chunk) {
+      this.port.postMessage(Float32Array.from(this.pending));
+      this.pending = [];
+    }
+    return true;
+  }
+}
+registerProcessor("pcm-capture", PcmCapture);
+`;
+
+export const createResampler = (ratio: number) => {
+  let position = 0;
+  let last = 0;
+  const lowPass = ratio > 1 ? createLowPass(ratio) : undefined;
+  return (input: Float32Array): Float32Array => {
+    if (input.length === 0) return new Float32Array(0);
+    const samples = lowPass ? input.slice() : input;
+    lowPass?.(samples);
+    const out: number[] = [];
+    while (position < samples.length - 1) {
+      const index = Math.floor(position);
+      const prev = index < 0 ? last : samples[index]!;
+      const next = samples[index + 1]!;
+      out.push(prev + (next - prev) * (position - index));
+      position += ratio;
+    }
+    position -= samples.length;
+    last = samples[samples.length - 1]!;
+    return Float32Array.from(out);
+  };
+};
+
+export const openMicrophone = async (options: {
+  sampleRate: number;
+  onChunk: (base64: string) => void;
+  signal?: AbortSignal;
+}): Promise<MicrophoneCapture> => {
+  let stream: MediaStream | undefined;
+  const stopTracks = () => {
+    stream?.getTracks().forEach((track) => track.stop());
+    stream = undefined;
+  };
+  const throwIfAborted = () => {
+    if (options.signal?.aborted) throw options.signal.reason;
+  };
+  let context: AudioContext | undefined;
+  try {
+    const openedStream = await raceWithAbortSignal(options.signal, async () => {
+      const acquired = await navigator.mediaDevices.getUserMedia({
+        audio: true,
+      });
+      stream = acquired;
+      if (options.signal?.aborted) {
+        stopTracks();
+        throw options.signal.reason;
+      }
+      return acquired;
+    });
+    throwIfAborted();
+    context = new AudioContext();
+    if (context.state === "suspended")
+      await raceWithAbortSignal(options.signal, () => context!.resume());
+    throwIfAborted();
+    const source = context.createMediaStreamSource(openedStream);
+    const analyser = context.createAnalyser();
+    analyser.fftSize = 256;
+    source.connect(analyser);
+
+    let muted = false;
+    let closed = false;
+    let closing: Promise<void> | undefined;
+    const push = (samples: Float32Array) => {
+      if (!muted && !closed && samples.length)
+        options.onChunk(encodePcm16(samples));
+    };
+
+    let node: AudioNode | undefined;
+    let detach = () => {};
+    if (context.audioWorklet) {
+      const url = URL.createObjectURL(
+        new Blob([WORKLET_SOURCE], { type: "text/javascript" }),
+      );
+      try {
+        await raceWithAbortSignal(options.signal, () =>
+          context!.audioWorklet.addModule(url),
+        );
+        throwIfAborted();
+        const worklet = new AudioWorkletNode(context, "pcm-capture", {
+          processorOptions: { targetRate: options.sampleRate },
+        });
+        worklet.port.onmessage = (event) => push(event.data as Float32Array);
+        detach = () => {
+          worklet.port.onmessage = null;
+        };
+        node = worklet;
+      } catch {
+        // A CSP can block blob worklets even when AudioWorklet is available.
+        throwIfAborted();
+      } finally {
+        URL.revokeObjectURL(url);
+      }
+    }
+    if (!node) {
+      const resample = createResampler(context.sampleRate / options.sampleRate);
+      const processor = context.createScriptProcessor(1024, 1, 1);
+      processor.onaudioprocess = (event) =>
+        push(resample(event.inputBuffer.getChannelData(0)));
+      detach = () => {
+        processor.onaudioprocess = null;
+      };
+      node = processor;
+    }
+    analyser.connect(node);
+    node.connect(context.destination);
+    const opened = context;
+
+    return {
+      setMuted: (value) => {
+        muted = value;
+        openedStream.getAudioTracks().forEach((track) => {
+          track.enabled = !value;
+        });
+      },
+      level: () => readLevel(analyser),
+      close: () => {
+        if (closing) return closing;
+        closed = true;
+        detach();
+        node.disconnect();
+        source.disconnect();
+        stopTracks();
+        return (closing = opened.close());
+      },
+    };
+  } catch (error) {
+    stopTracks();
+    await context?.close().catch(() => {});
+    throw error;
+  }
+};
+
+export type PcmPlayer = {
+  enqueue: (base64: string) => void;
+  clear: () => void;
+  isPlaying: () => boolean;
+  level: () => number;
+  close: () => Promise<void>;
+};
+
+const IDLE_GRACE_MS = 100;
+
+export const createPcmPlayer = (options: {
+  sampleRate: number;
+  onIdle?: () => void;
+}): PcmPlayer => {
+  const context = new AudioContext({ sampleRate: options.sampleRate });
+  const analyser = context.createAnalyser();
+  analyser.fftSize = 256;
+  analyser.connect(context.destination);
+  const sources = new Set<AudioBufferSourceNode>();
+  let cursor = 0;
+  let closing: Promise<void> | undefined;
+  let idleTimer: ReturnType<typeof setTimeout> | undefined;
+  const scheduleIdle = () => {
+    clearTimeout(idleTimer);
+    idleTimer = setTimeout(() => {
+      if (sources.size === 0) options.onIdle?.();
+    }, IDLE_GRACE_MS);
+  };
+
+  return {
+    enqueue: (base64) => {
+      const samples = decodePcm16(base64);
+      if (samples.length === 0) return;
+      const buffer = context.createBuffer(
+        1,
+        samples.length,
+        options.sampleRate,
+      );
+      buffer.copyToChannel(samples, 0);
+      const source = context.createBufferSource();
+      source.buffer = buffer;
+      source.connect(analyser);
+      const start = Math.max(context.currentTime, cursor);
+      source.start(start);
+      cursor = start + buffer.duration;
+      sources.add(source);
+      source.onended = () => {
+        sources.delete(source);
+        if (sources.size === 0) scheduleIdle();
+      };
+      clearTimeout(idleTimer);
+      if (context.state === "suspended") void context.resume().catch(() => {});
+    },
+    clear: () => {
+      clearTimeout(idleTimer);
+      for (const source of sources) {
+        source.onended = null;
+        source.stop();
+      }
+      sources.clear();
+      cursor = 0;
+    },
+    isPlaying: () => sources.size > 0,
+    level: () => readLevel(analyser),
+    close: () => {
+      if (closing) return closing;
+      clearTimeout(idleTimer);
+      for (const source of sources) {
+        source.onended = null;
+        source.stop();
+      }
+      sources.clear();
+      return (closing = context.close());
+    },
+  };
+};

package/src/index.test.ts

@@ -0,0 +1,34 @@
+/// <reference types="node" />
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { expect, it } from "vitest";
+
+const PACKAGE_DIR = fileURLToPath(new URL("..", import.meta.url));
+
+// The built entry imports React through @assistant-ui/tap/react-shim. Vite
+// dev pre-bundles that shim with esbuild, whose CommonJS interop hands it
+// React's default export only; the loader below recreates that for the shim.
+const script = `
+import { registerHooks } from "node:module";
+registerHooks({
+  resolve(specifier, context, nextResolve) {
+    const resolved = nextResolve(specifier, context);
+    if (specifier !== "react" || !context.parentURL?.endsWith("/react-shim/index.js")) return resolved;
+    const source = "import React from " + JSON.stringify(resolved.url) + "; export default React;";
+    return { url: "data:text/javascript," + encodeURIComponent(source), shortCircuit: true };
+  },
+});
+const entry = await import("@assistant-ui/react");
+console.log(typeof entry.ComposerPrimitive.Root, typeof entry.CloudRendererHost);
+`;
+
+it("loads the public entry when the react-shim only sees React's default export", () => {
+  const result = spawnSync(
+    process.execPath,
+    [...process.execArgv, "--input-type=module", "--eval", script],
+    { cwd: PACKAGE_DIR, encoding: "utf8" },
+  );
+
+  expect(result.status, result.stderr).toBe(0);
+  expect(result.stdout.trim()).toBe("object function");
+});

package/src/index.ts

@@ -69,6 +69,10 @@ export {
   OpenAIRealtimeVoiceAdapter,
   type OpenAIRealtimeVoiceAdapterOptions,
 } from "./adapters/OpenAIRealtimeVoiceAdapter";
+export {
+  GeminiLiveVoiceAdapter,
+  type GeminiLiveVoiceAdapterOptions,
+} from "./adapters/GeminiLiveVoiceAdapter";
 export type {
   VoiceSessionControls,
   VoiceSessionHelpers,

package/src/mcp-apps/McpAppRenderer.composer.test.tsx

@@ -0,0 +1,111 @@
+// @vitest-environment jsdom
+import { act, render, waitFor } from "@testing-library/react";
+import { resource, useResource } from "@assistant-ui/tap";
+import {
+  AssistantRuntimeProvider,
+  MessageByIndexProvider,
+  useExternalStoreRuntime,
+  type ToolCallMessagePartProps,
+} from "@assistant-ui/core/react";
+import type { ThreadMessageLike } from "@assistant-ui/core";
+import { expect, it, vi } from "vitest";
+import type { AssistantRuntime } from "../index";
+import type { McpAppBridgeHandlers, McpAppsHost } from "./types";
+
+const { framePropsMock } = vi.hoisted(() => ({ framePropsMock: vi.fn() }));
+
+vi.mock("./app-frame", () => ({
+  McpAppFrame: (props: unknown) => {
+    framePropsMock(props);
+    return null;
+  },
+}));
+
+import { McpAppRenderer } from "./McpAppRenderer";
+
+const Host = resource(({ host }: { host: McpAppsHost }) => host);
+
+const host: McpAppsHost = {
+  loadResource: vi.fn(async ({ uri }) => ({
+    uri,
+    mimeType: "text/html;profile=mcp-app" as const,
+    html: "",
+  })),
+  callTool: vi.fn(),
+  readResource: vi.fn(),
+  listResources: vi.fn(),
+};
+
+const part: ToolCallMessagePartProps = {
+  type: "tool-call",
+  toolCallId: "call-1",
+  toolName: "search",
+  args: {},
+  argsText: "{}",
+  status: { type: "complete" },
+  mcp: { app: { resourceUri: "ui://example/search" } },
+  addResult: vi.fn(),
+  resume: vi.fn(),
+  respondToApproval: vi.fn(),
+};
+
+const messages: ThreadMessageLike[] = [
+  { role: "assistant", content: "Here is the app" },
+];
+
+const AppPart = () => {
+  const renderer = useResource(McpAppRenderer({ host: Host({ host }) }));
+  const Renderer = renderer.render;
+  return <Renderer {...part} />;
+};
+
+let runtime!: AssistantRuntime;
+
+const App = () => {
+  runtime = useExternalStoreRuntime({
+    messages,
+    convertMessage: (message: ThreadMessageLike) => message,
+    onNew: async () => {},
+  });
+  return (
+    <AssistantRuntimeProvider runtime={runtime}>
+      <MessageByIndexProvider index={0}>
+        <AppPart />
+      </MessageByIndexProvider>
+    </AssistantRuntimeProvider>
+  );
+};
+
+const renderApp = async () => {
+  framePropsMock.mockClear();
+  render(<App />);
+  await waitFor(() => expect(framePropsMock).toHaveBeenCalled());
+  const handlers = framePropsMock.mock.lastCall?.[0]
+    .handlers as McpAppBridgeHandlers;
+  return handlers;
+};
+
+it("fills the thread composer from an MCP App message inside a message", async () => {
+  const handlers = await renderApp();
+
+  let result: unknown;
+  act(() => {
+    result = handlers.sendMessage?.({ text: "Open Grade 5 Math" });
+  });
+
+  expect(result).toEqual({ ok: true });
+  expect(runtime.thread.composer.getState().text).toBe("Open Grade 5 Math");
+});
+
+it("keeps the user's draft when an MCP App message arrives", async () => {
+  const handlers = await renderApp();
+  act(() => runtime.thread.composer.setText("Show my classes"));
+
+  act(() => {
+    handlers.sendMessage?.({ text: "Open Grade 5 Math" });
+  });
+
+  expect(runtime.thread.composer.getState().text).toBe(
+    "Show my classes Open Grade 5 Math",
+  );
+});

package/src/mcp-apps/McpAppRenderer.test.tsx

@@ -16,15 +16,22 @@ import type {
   McpAppsRemoteHostOptions,
 } from "./types";
 
-const { appendMock, framePropsMock } = vi.hoisted(() => ({
+const { appendMock, setTextMock, framePropsMock } = vi.hoisted(() => ({
   appendMock: vi.fn(),
+  setTextMock: vi.fn(),
   framePropsMock: vi.fn(),
 }));
 
 vi.mock("@assistant-ui/store", async (importOriginal) => ({
   ...(await importOriginal()),
   useAui: () => ({
-    thread: { append: appendMock },
+    thread: {
+      append: appendMock,
+      composer: () => ({
+        setText: setTextMock,
+        getState: () => ({ text: "" }),
+      }),
+    },
   }),
 }));
 
@@ -190,6 +197,7 @@ function RemoteHarness({
 describe("McpAppRenderer", () => {
   beforeEach(() => {
     appendMock.mockReset();
+    setTextMock.mockReset();
     framePropsMock.mockReset();
   });
 
@@ -676,7 +684,7 @@ describe("McpAppRenderer", () => {
     expect(listResources).toHaveBeenCalledWith(params);
   });
 
-  it("appends MCP Apps message content in the default sendMessage handler", async () => {
+  it("puts MCP Apps text content in the composer without appending", async () => {
     render(<Harness host={loadingHost()} />);
     await waitFor(() => expect(framePropsMock).toHaveBeenCalled());
     const handlers = framePropsMock.mock.lastCall?.[0]
@@ -692,19 +700,13 @@ describe("McpAppRenderer", () => {
         ],
       }),
     ).toEqual({ ok: true });
-    expect(appendMock).toHaveBeenCalledWith({
-      content: [
-        { type: "text", text: "Open Grade 5 Math" },
-        { type: "text", text: "Period 4" },
-      ],
-    });
+    expect(setTextMock).toHaveBeenCalledWith("Open Grade 5 Math\nPeriod 4");
+    expect(appendMock).not.toHaveBeenCalled();
 
     expect(handlers.sendMessage?.({ text: "typed instead" })).toEqual({
       ok: true,
     });
-    expect(appendMock).toHaveBeenLastCalledWith({
-      content: [{ type: "text", text: "typed instead" }],
-    });
+    expect(setTextMock).toHaveBeenLastCalledWith("typed instead");
 
     expect(
       handlers.sendMessage?.({
@@ -722,7 +724,8 @@ describe("McpAppRenderer", () => {
       ok: false,
       reason: "unrecognised params shape",
     });
-    expect(appendMock).toHaveBeenCalledTimes(2);
+    expect(setTextMock).toHaveBeenCalledTimes(2);
+    expect(appendMock).not.toHaveBeenCalled();
   });
 
   it("extracts sparse message content without traversing holes", async () => {
@@ -755,9 +758,8 @@ describe("McpAppRenderer", () => {
     }
 
     expect(visitedHole).not.toHaveBeenCalled();
-    expect(appendMock).toHaveBeenCalledWith({
-      content: [{ type: "text", text: "sparse prompt" }],
-    });
+    expect(setTextMock).toHaveBeenCalledExactlyOnceWith("sparse prompt");
+    expect(appendMock).not.toHaveBeenCalled();
   });
 
   it("ignores non-index message content properties after structured clone", async () => {
@@ -781,13 +783,10 @@ describe("McpAppRenderer", () => {
       ).toEqual({ ok: true });
     }
 
-    expect(appendMock).toHaveBeenCalledTimes(2);
-    expect(appendMock).toHaveBeenNthCalledWith(1, {
-      content: [{ type: "text", text: "message text" }],
-    });
-    expect(appendMock).toHaveBeenNthCalledWith(2, {
-      content: [{ type: "text", text: "message text" }],
-    });
+    expect(setTextMock).toHaveBeenCalledTimes(2);
+    expect(setTextMock).toHaveBeenNthCalledWith(1, "message text");
+    expect(setTextMock).toHaveBeenNthCalledWith(2, "message text");
+    expect(appendMock).not.toHaveBeenCalled();
   });
 
   it("uses caller UI handlers and keeps data-plane handlers on the host", async () => {
@@ -838,6 +837,8 @@ describe("McpAppRenderer", () => {
     expect(updateModelContext).toHaveBeenCalledWith({ text: "context" });
     expect(openLink).toHaveBeenCalledWith({ url: "https://example.com" });
     expect(sendMessage).toHaveBeenCalledWith({ text: "hello" });
+    expect(setTextMock).not.toHaveBeenCalled();
+    expect(appendMock).not.toHaveBeenCalled();
     expect(onInitialized).toHaveBeenCalledOnce();
     expect(host.callTool).toHaveBeenCalledWith({ name: "search" });
     expect(host.readResource).toHaveBeenCalledWith({ uri: "ui://resource" });

package/src/mcp-apps/McpAppRenderer.tsx

@@ -277,7 +277,15 @@ function InlineRenderer({
           if ("reason" in result) {
             return { isError: true, ok: false, reason: result.reason };
           }
-          aui.thread.append({ content: result.content });
+          const composer = aui.thread.composer();
+          composer.setText(
+            [
+              composer.getState().text,
+              result.content.map((part) => part.text).join("\n"),
+            ]
+              .filter((part) => part.trim())
+              .join(" "),
+          );
           return { ok: true };
         }),
       callTool: (params) =>