@agent-native/toolkit 0.221.0-same-host-mcp-20261027-20261027081850 → 0.221.0

package/src/composer/useRealtimeVoiceMode.session.spec.tsx

@@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import {
   realtimeVoiceTranscriptRegistry,
   type RealtimeVoiceTranscriptMessage,
+  type RealtimeVoiceWorkUpdate,
 } from "./realtime-voice-transcript.js";
 import { ComposerRuntimeAdaptersProvider } from "./runtime-adapters.js";
 import {
@@ -14,16 +15,20 @@ import {
   RealtimeVoiceModeProvider,
   useRealtimeVoiceMode,
   type RealtimeVoiceModeApi,
+  type RealtimeVoiceStartOptions,
 } from "./useRealtimeVoiceMode.js";
 
 interface FakeChannel {
   onmessage?: (event: { data: string }) => void;
   close: () => void;
+  readyState?: string;
+  send?: (data: string) => void;
 }
 
 let channel: FakeChannel;
 let track: { enabled: boolean; stop: () => void; onended: unknown };
 let audio: HTMLAudioElement | undefined;
+let setAppState: ReturnType<typeof vi.fn>;
 
 class FakePeerConnection {
   connectionState = "new";
@@ -46,6 +51,7 @@ beforeEach(() => {
   channel = { close: vi.fn() };
   track = { enabled: true, stop: vi.fn(), onended: null };
   audio = undefined;
+  setAppState = vi.fn();
   vi.stubGlobal("RTCPeerConnection", FakePeerConnection);
   Object.defineProperty(navigator, "mediaDevices", {
     configurable: true,
@@ -75,6 +81,11 @@ afterEach(() => {
 
 async function startSession(
   readAppState: (key: string) => unknown = () => undefined,
+  startOptions?: RealtimeVoiceStartOptions,
+  trackEvent: (
+    name: string,
+    properties: Record<string, unknown>,
+  ) => void = () => undefined,
 ) {
   let api: RealtimeVoiceModeApi | undefined;
   function Probe() {
@@ -89,8 +100,9 @@ async function startSession(
         adapters={{
           voice: {
             readAppState: readAppState as never,
-            setAppState: () => undefined,
+            setAppState: setAppState as never,
             getBrowserTabId: () => "tab-1",
+            trackEvent,
           },
         }}
       >
@@ -101,7 +113,7 @@ async function startSession(
     );
   });
   await act(async () => {
-    void api!.start();
+    void api!.start(startOptions);
     await new Promise((resolve) => setTimeout(resolve, 0));
   });
   return {
@@ -116,6 +128,381 @@ async function startSession(
 }
 
 describe("realtime voice session controls", () => {
+  it("groups one reply's tool calls into one piece of work", async () => {
+    const work: RealtimeVoiceWorkUpdate[] = [];
+    const unregister = realtimeVoiceTranscriptRegistry.register({
+      threadId: "thread-work",
+      active: true,
+      append: () => true,
+      appendWork: (update) => {
+        work.push(update);
+        return true;
+      },
+    });
+    const results: Array<(value: Response) => void> = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(() => new Promise<Response>((resolve) => results.push(resolve))),
+    );
+    const session = await startSession();
+    const resolveCall = async (callId: string) => {
+      await act(async () => {
+        results.shift()?.(
+          Response.json({ callId, status: "completed", output: "{}" }),
+        );
+        await new Promise((done) => setTimeout(done, 0));
+      });
+    };
+
+    await session.send({ type: "response.created" });
+    for (const [callId, name] of [
+      ["call-search", "web-search"],
+      ["call-read", "web-request"],
+    ] as const) {
+      await session.send({
+        type: "response.function_call_arguments.done",
+        call_id: callId,
+        name,
+        arguments: "{}",
+      });
+    }
+    // The response that asked for the tools is not the end of the reply.
+    await session.send({
+      type: "response.done",
+      response: { status: "completed", output: [] },
+    });
+    expect(session.api().currentToolName).toBe("web-request");
+    await resolveCall("call-search");
+    expect(session.api().currentToolName).toBe("web-request");
+    await resolveCall("call-read");
+    expect(session.api().currentToolName).toBeNull();
+    expect(work.every((update) => !update.completedAt)).toBe(true);
+
+    await session.send({ type: "response.created" });
+    await session.send({
+      type: "response.done",
+      response: { status: "completed", output: [] },
+    });
+
+    const last = work.at(-1);
+    expect(new Set(work.map((update) => update.id))).toEqual(
+      new Set(["call-search"]),
+    );
+    expect(last?.calls.map((call) => [call.id, call.status])).toEqual([
+      ["call-search", "completed"],
+      ["call-read", "completed"],
+    ]);
+    expect(last?.completedAt).toBe(last?.calls.at(-1)?.completedAt);
+    unregister();
+    session.unmount();
+  });
+
+  it("groups a GPT-Live delegation's tool calls into one piece of work", async () => {
+    const work: RealtimeVoiceWorkUpdate[] = [];
+    const unregister = realtimeVoiceTranscriptRegistry.register({
+      threadId: "thread-live-work",
+      active: true,
+      append: () => true,
+      appendWork: (update) => {
+        work.push(update);
+        return true;
+      },
+    });
+    const results: Array<(value: Response) => void> = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(() => new Promise<Response>((resolve) => results.push(resolve))),
+    );
+    const session = await startSession();
+    const nested = (delegationId: string, event: Record<string, unknown>) =>
+      session.send({
+        type: "response.event",
+        delegation_id: delegationId,
+        event,
+      });
+    // Live response snapshots never carry their output.
+    const doneSnapshot = {
+      type: "response.done",
+      response: { status: "completed" },
+    };
+    const callResponse = async (delegationId: string, callId: string) => {
+      await nested(delegationId, { type: "response.created" });
+      await nested(delegationId, {
+        type: "response.output_item.done",
+        item: {
+          type: "function_call",
+          call_id: callId,
+          name: "web-search",
+          arguments: "{}",
+        },
+      });
+      await nested(delegationId, doneSnapshot);
+      await act(async () => {
+        results.shift()?.(
+          Response.json({ callId, status: "completed", output: "{}" }),
+        );
+        await new Promise((done) => setTimeout(done, 0));
+      });
+    };
+
+    await callResponse("delegation-1", "call-1");
+    await callResponse("delegation-1", "call-2");
+    await callResponse("delegation-1", "call-3");
+    expect(work.every((update) => !update.completedAt)).toBe(true);
+    await nested("delegation-1", { type: "response.created" });
+    await nested("delegation-1", doneSnapshot);
+    expect(work.at(-1)).toMatchObject({ id: "call-1" });
+    expect(work.at(-1)?.calls.map((call) => call.id)).toEqual([
+      "call-1",
+      "call-2",
+      "call-3",
+    ]);
+    expect(work.at(-1)?.completedAt).toBeTruthy();
+
+    // The next request is a new delegation and new work.
+    await callResponse("delegation-2", "call-4");
+    expect(work.at(-1)).toMatchObject({ id: "call-4" });
+    unregister();
+    session.unmount();
+  });
+
+  it.each([
+    ["approved", true],
+    ["declined", false],
+  ])(
+    "holds a call that needs approval until the user has %s it",
+    async (decision, approved) => {
+      const callId = `call-delete-${decision}`;
+      const work: RealtimeVoiceWorkUpdate[] = [];
+      const unregister = realtimeVoiceTranscriptRegistry.register({
+        threadId: `thread-approval-${String(approved)}`,
+        active: true,
+        append: () => true,
+        appendWork: (update) => {
+          work.push(update);
+          return true;
+        },
+      });
+      const sent: Array<Record<string, unknown>> = [];
+      channel = {
+        close: vi.fn(),
+        readyState: "open",
+        send: (data: string) => sent.push(JSON.parse(data)),
+      } as FakeChannel;
+      const requests: Array<Record<string, unknown>> = [];
+      vi.stubGlobal(
+        "fetch",
+        vi.fn(async (_url: string, init?: RequestInit) => {
+          const body = JSON.parse(String(init?.body)) as Record<
+            string,
+            unknown
+          >;
+          requests.push(body);
+          return Response.json(
+            body.approvalId || body.declinedApprovalId
+              ? { callId: body.callId, status: "completed", output: "deleted" }
+              : // No key: arguments too long for one still get an approval.
+                {
+                  callId: body.callId,
+                  status: "approval_required",
+                  output: "Awaiting approval",
+                  approvalId: "approval-1",
+                },
+          );
+        }),
+      );
+      const session = await startSession();
+      await session.send({ type: "response.created" });
+      await session.send({
+        type: "response.function_call_arguments.done",
+        call_id: callId,
+        name: "delete-slide",
+        arguments: JSON.stringify({ slideId: "s1" }),
+      });
+      await act(async () => {
+        await new Promise((done) => setTimeout(done, 0));
+      });
+
+      expect(session.api().approvals).toEqual([
+        { callId, toolName: "delete-slide" },
+      ]);
+      expect(work.at(-1)?.calls.map((call) => call.status)).toEqual([
+        "running",
+      ]);
+      sent.length = 0;
+
+      await act(async () => {
+        await session.api().resolveApproval(callId, approved);
+      });
+
+      expect(session.api().approvals).toEqual([]);
+      expect(requests.slice(1)).toEqual([
+        approved
+          ? expect.objectContaining({
+              name: "delete-slide",
+              args: { slideId: "s1" },
+              approvalId: "approval-1",
+            })
+          : expect.objectContaining({ declinedApprovalId: "approval-1" }),
+      ]);
+      expect(work.at(-1)?.calls.map((call) => call.status)).toEqual([
+        approved ? "completed" : "cancelled",
+      ]);
+      // A session that has not connected yet speaks GPT-Live.
+      expect(sent.map((event) => event.type)).toEqual([
+        "response.item.create",
+        "response.create",
+      ]);
+      expect(JSON.stringify(sent[0])).toContain(
+        approved ? "The user approved delete-slide" : "The user declined",
+      );
+      unregister();
+      await session.unmount();
+    },
+  );
+
+  it("asks again when an approved call's approval no longer applies", async () => {
+    const work: RealtimeVoiceWorkUpdate[] = [];
+    const unregister = realtimeVoiceTranscriptRegistry.register({
+      threadId: "thread-approval-renewed",
+      active: true,
+      append: () => true,
+      appendWork: (update) => {
+        work.push(update);
+        return true;
+      },
+    });
+    const sent: Array<Record<string, unknown>> = [];
+    channel = {
+      close: vi.fn(),
+      readyState: "open",
+      send: (data: string) => sent.push(JSON.parse(data)),
+    } as FakeChannel;
+    const requests: Array<Record<string, unknown>> = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (_url: string, init?: RequestInit) => {
+        const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
+        requests.push(body);
+        // The first approval expired, so the approved retry asks again.
+        return Response.json(
+          body.approvalId === "approval-2"
+            ? { callId: body.callId, status: "completed", output: "deleted" }
+            : {
+                callId: body.callId,
+                status: "approval_required",
+                output: "Awaiting approval",
+                approvalKey: "delete-slide:key",
+                approvalId: body.approvalId ? "approval-2" : "approval-1",
+              },
+        );
+      }),
+    );
+    const session = await startSession();
+    await session.send({ type: "response.created" });
+    await session.send({
+      type: "response.function_call_arguments.done",
+      call_id: "call-renewed",
+      name: "delete-slide",
+      arguments: JSON.stringify({ slideId: "s1" }),
+    });
+    await act(async () => {
+      await new Promise((done) => setTimeout(done, 0));
+    });
+    sent.length = 0;
+
+    await act(async () => {
+      await session.api().resolveApproval("call-renewed", true);
+    });
+    expect(session.api().approvals).toEqual([
+      { callId: "call-renewed", toolName: "delete-slide" },
+    ]);
+    expect(work.at(-1)?.calls.map((call) => call.status)).toEqual(["running"]);
+    expect(JSON.stringify(sent[0])).toContain("asked for approval again");
+
+    await act(async () => {
+      await session.api().resolveApproval("call-renewed", true);
+    });
+    expect(requests.at(-1)).toMatchObject({ approvalId: "approval-2" });
+    expect(work.at(-1)?.calls.map((call) => call.status)).toEqual([
+      "completed",
+    ]);
+    unregister();
+    await session.unmount();
+  });
+
+  it("records a tool call that teardown aborted as cancelled", async () => {
+    const work: RealtimeVoiceWorkUpdate[] = [];
+    const unregister = realtimeVoiceTranscriptRegistry.register({
+      threadId: "thread-teardown",
+      active: true,
+      append: () => true,
+      appendWork: (update) => {
+        work.push(update);
+        return true;
+      },
+    });
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        (_url: string, init?: RequestInit) =>
+          new Promise<Response>((_resolve, reject) => {
+            init?.signal?.addEventListener("abort", () =>
+              reject(new DOMException("Aborted", "AbortError")),
+            );
+          }),
+      ),
+    );
+    const session = await startSession();
+    await session.send({ type: "response.created" });
+    await session.send({
+      type: "response.function_call_arguments.done",
+      call_id: "call-edit",
+      name: "web-search",
+      arguments: "{}",
+    });
+
+    await act(async () => {
+      session.api().end();
+      await new Promise((done) => setTimeout(done, 0));
+    });
+
+    expect(work.at(-1)?.calls.map((call) => [call.id, call.status])).toEqual([
+      ["call-edit", "cancelled"],
+    ]);
+    unregister();
+    session.unmount();
+  });
+
+  it("reports each turn's timing without the words spoken", async () => {
+    const trackEvent = vi.fn();
+    const session = await startSession(undefined, undefined, trackEvent);
+
+    await session.send({ type: "input_audio_buffer.speech_started" });
+    await session.send({ type: "input_audio_buffer.speech_stopped" });
+    await session.send({ type: "output_audio_buffer.started" });
+    await session.send({
+      type: "response.output_audio_transcript.delta",
+      item_id: "assistant-1",
+      delta: "Opening the secret plans",
+    });
+    expect(trackEvent).not.toHaveBeenCalled();
+    await session.send({ type: "input_audio_buffer.speech_started" });
+
+    expect(trackEvent).toHaveBeenCalledTimes(1);
+    const [name, properties] = trackEvent.mock.calls[0]!;
+    expect(name).toBe("realtime_voice_turn");
+    expect(properties).toMatchObject({
+      turn_index: 0,
+      tool_calls: 0,
+      latency_clock: "browser",
+      spoke_again_after_edit: null,
+    });
+    expect(properties.speech_to_audio_ms).toEqual(expect.any(Number));
+    expect(JSON.stringify(properties)).not.toContain("secret");
+    session.unmount();
+  });
+
   it("mutes the microphone track and the assistant's playback", async () => {
     const session = await startSession();
     expect(session.api().state).toBe("connecting");
@@ -199,6 +586,103 @@ describe("realtime voice session controls", () => {
     session.unmount();
   });
 
+  it("streams into the thread it was started for until that thread is shown", async () => {
+    const onScreen: RealtimeVoiceTranscriptMessage[] = [];
+    const unregisterOnScreen = realtimeVoiceTranscriptRegistry.register({
+      threadId: "thread-on-screen",
+      active: true,
+      append: (message) => {
+        onScreen.push(message);
+        return true;
+      },
+    });
+    const session = await startSession(undefined, {
+      threadId: "thread-handoff",
+    });
+
+    await session.send({
+      type: "conversation.item.input_audio_transcription.completed",
+      item_id: "user-handoff",
+      transcript: "Plan my week.",
+    });
+    expect(onScreen).toEqual([]);
+
+    const handedOff: RealtimeVoiceTranscriptMessage[] = [];
+    const unregisterHandoff = realtimeVoiceTranscriptRegistry.register({
+      threadId: "thread-handoff",
+      active: true,
+      append: (message) => {
+        handedOff.push(message);
+        return true;
+      },
+    });
+
+    expect(
+      handedOff.map(({ threadId, role, text, status }) => ({
+        threadId,
+        role,
+        text,
+        status,
+      })),
+    ).toEqual([
+      {
+        threadId: "thread-handoff",
+        role: "user",
+        text: "Plan my week.",
+        status: "complete",
+      },
+    ]);
+    unregisterHandoff();
+    unregisterOnScreen();
+    session.unmount();
+  });
+
+  it("saves the app's default level as not chosen so Settings can show it", async () => {
+    vi.stubGlobal(
+      "RTCPeerConnection",
+      class extends FakePeerConnection {
+        override createOffer() {
+          return Promise.resolve({ type: "offer", sdp: "offer-sdp" });
+        }
+        setLocalDescription() {
+          return Promise.resolve();
+        }
+        setRemoteDescription() {
+          return Promise.resolve();
+        }
+      },
+    );
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response("answer-sdp", {
+            headers: {
+              "X-Agent-Native-Realtime-Protocol": "live",
+              "X-Agent-Native-Realtime-Intelligence": "balanced",
+            },
+          }),
+      ),
+    );
+    const session = await startSession();
+    await act(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    });
+
+    expect(session.api().preferences).toMatchObject({
+      intelligence: "balanced",
+      intelligenceChosen: false,
+    });
+    expect(setAppState).toHaveBeenCalledWith(
+      REALTIME_VOICE_PREFERENCES_KEY,
+      expect.objectContaining({
+        intelligence: "balanced",
+        intelligenceChosen: false,
+      }),
+    );
+    session.unmount();
+  });
+
   it("starts from the voice chat choices saved in Settings", async () => {
     window.localStorage.setItem(
       "agent-native:realtime-voice-microphone",
@@ -216,6 +700,7 @@ describe("realtime voice session controls", () => {
       language: "ja",
       intelligence: "deep",
       voice: "cedar",
+      intelligenceChosen: true,
     });
     expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledWith({
       audio: expect.objectContaining({ deviceId: { ideal: "studio" } }),