@kuralle-syrinx/realtime 2.1.0

package/src/from-openai-realtime.test.ts

@@ -0,0 +1,447 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+
+import type { ManagedSocket, SocketData, SocketFactory } from "@kuralle-syrinx/ws";
+
+import type { RealtimeEvent } from "./realtime-adapter.js";
+import { base64ToBytes, bytesToBase64, fromOpenAIRealtime } from "./from-openai-realtime.js";
+
+interface MockSocketHarness {
+  readonly factory: SocketFactory;
+  readonly sent: string[];
+  inject(msg: Record<string, unknown>): void;
+}
+
+function createMockSocketHarness(): MockSocketHarness {
+  const sent: string[] = [];
+  let messageHandler: ((data: SocketData, isBinary: boolean) => void) | null = null;
+
+  const socket: ManagedSocket = {
+    get isOpen() {
+      return true;
+    },
+    send: (data: SocketData) => {
+      sent.push(typeof data === "string" ? data : "");
+    },
+    keepAlivePing: () => {},
+    verify: async () => true,
+    dispose: () => {},
+    onOpen: (handler) => {
+      queueMicrotask(() => handler());
+    },
+    onMessage: (handler) => {
+      messageHandler = handler;
+    },
+    onClose: () => {},
+    onError: () => {},
+  };
+
+  return {
+    factory: () => socket,
+    sent,
+    inject: (msg) => messageHandler?.(JSON.stringify(msg), false),
+  };
+}
+
+async function collectEvents(
+  events: AsyncIterable<RealtimeEvent>,
+  max = 8,
+): Promise<RealtimeEvent[]> {
+  const out: RealtimeEvent[] = [];
+  for await (const event of events) {
+    out.push(event);
+    if (out.length >= max) break;
+  }
+  return out;
+}
+
+async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 5));
+  }
+  throw new Error("Timed out waiting for condition");
+}
+
+describe("fromOpenAIRealtime", () => {
+  it("emits exact client events for open, audio, cancel, and tool result", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+      tools: [
+        {
+          name: "consult_knowledge",
+          description: "Answer knowledge questions.",
+          parameters: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
+        },
+      ],
+    });
+
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    const sessionUpdate = JSON.parse(mock.sent[0]!) as Record<string, unknown>;
+    expect(sessionUpdate).toEqual({
+      type: "session.update",
+      session: {
+        type: "realtime",
+        model: "gpt-realtime-2",
+        output_modalities: ["audio"],
+        audio: {
+          input: {
+            format: { type: "audio/pcm", rate: 24000 },
+            turn_detection: { type: "semantic_vad" },
+          },
+          output: {
+            format: { type: "audio/pcm", rate: 24000 },
+            voice: "marin",
+          },
+        },
+        tools: [
+          {
+            type: "function",
+            name: "consult_knowledge",
+            description: "Answer knowledge questions.",
+            parameters: {
+              type: "object",
+              properties: { query: { type: "string" } },
+              required: ["query"],
+            },
+          },
+        ],
+        tool_choice: "auto",
+      },
+    });
+
+    const pcm = new Uint8Array([0, 1, 2, 3]);
+    adapter.sendAudio(pcm);
+    expect(JSON.parse(mock.sent[1]!)).toEqual({
+      type: "input_audio_buffer.append",
+      audio: bytesToBase64(pcm),
+    });
+
+    mock.inject({ type: "response.created" });
+    mock.inject({
+      type: "response.output_item.added",
+      item: { type: "message", id: "item_assistant_1" },
+    });
+    adapter.cancelResponse(420);
+    expect(JSON.parse(mock.sent[2]!)).toEqual({ type: "response.cancel" });
+    expect(JSON.parse(mock.sent[3]!)).toEqual({
+      type: "conversation.item.truncate",
+      item_id: "item_assistant_1",
+      content_index: 0,
+      audio_end_ms: 420,
+    });
+
+    adapter.injectToolResult("call_abc", "Late Add Petition required.");
+    expect(JSON.parse(mock.sent[4]!)).toEqual({
+      type: "conversation.item.create",
+      item: {
+        type: "function_call_output",
+        call_id: "call_abc",
+        output: "Late Add Petition required.",
+      },
+    });
+    expect(mock.sent).toHaveLength(5);
+
+    mock.inject({ type: "response.done", response: {} });
+    expect(JSON.parse(mock.sent[5]!)).toEqual({ type: "response.create" });
+  });
+
+  it("normalizes provider server events into RealtimeEvent", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+    });
+
+    const eventsTask = collectEvents(adapter.events, 8);
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    const audioBytes = new Uint8Array([9, 10, 11, 12]);
+    mock.inject({
+      type: "response.created",
+    });
+    mock.inject({
+      type: "response.output_audio.delta",
+      delta: bytesToBase64(audioBytes),
+    });
+    mock.inject({ type: "input_audio_buffer.speech_started" });
+    mock.inject({
+      type: "response.output_audio_transcript.delta",
+      delta: "Let me ",
+    });
+    mock.inject({
+      type: "response.output_audio_transcript.done",
+      transcript: "Let me check that.",
+    });
+    mock.inject({
+      type: "response.done",
+      response: {
+        output: [
+          {
+            type: "function_call",
+            call_id: "call_123",
+            name: "consult_knowledge",
+            arguments: JSON.stringify({ query: "late add biology" }),
+          },
+        ],
+      },
+    });
+    mock.inject({
+      type: "error",
+      error: { message: "rate limited", code: "rate_limit_exceeded" },
+    });
+
+    const events = await eventsTask;
+    expect(events.slice(0, 7)).toEqual([
+      { type: "response_started" },
+      { type: "audio", pcm16: audioBytes, sampleRateHz: 24000 },
+      { type: "speech_started" },
+      { type: "transcript", role: "assistant", text: "Let me ", final: false },
+      { type: "transcript", role: "assistant", text: "Let me check that.", final: true },
+      {
+        type: "tool_call",
+        toolId: "call_123",
+        toolName: "consult_knowledge",
+        args: { query: "late add biology" },
+      },
+      { type: "response_done" },
+    ]);
+    expect(events[7]).toMatchObject({ type: "error", recoverable: true });
+    if (events[7]?.type === "error") {
+      expect(events[7].cause.message).toBe("rate limited");
+    }
+  });
+
+  it("exposes gpt-realtime-2 capability flags", () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+    });
+    expect(adapter.caps).toEqual({
+      inputSampleRateHz: 24000,
+      outputSampleRateHz: 24000,
+      supportsConcurrentToolAudio: true,
+      supportsTruncate: true,
+      emitsServerSpeechStarted: true,
+    });
+  });
+
+  it("R-04: base64 encode/decode works without Buffer or process", async () => {
+    const savedBuffer = (globalThis as { Buffer?: unknown }).Buffer;
+    const savedProcess = (globalThis as { process?: unknown }).process;
+    delete (globalThis as { Buffer?: unknown }).Buffer;
+    delete (globalThis as { process?: unknown }).process;
+
+    try {
+      const pcm = new Uint8Array([0, 1, 2, 3, 255, 128]);
+      const encoded = bytesToBase64(pcm);
+      expect(base64ToBytes(encoded)).toEqual(pcm);
+
+      const mock = createMockSocketHarness();
+      const adapter = fromOpenAIRealtime({
+        apiKey: "test-key",
+        socketFactory: mock.factory,
+        url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+      });
+
+      const openTask = adapter.open(new AbortController().signal);
+      await waitFor(() => mock.sent.length > 0);
+      mock.inject({ type: "session.updated" });
+      await openTask;
+
+      adapter.sendAudio(pcm);
+      const sent = JSON.parse(mock.sent[1]!) as { audio: string };
+      expect(sent.audio).toBe(encoded);
+    } finally {
+      if (savedBuffer !== undefined) (globalThis as { Buffer?: unknown }).Buffer = savedBuffer;
+      if (savedProcess !== undefined) (globalThis as { process?: unknown }).process = savedProcess;
+    }
+  });
+
+  it("R-06: forwards optional session config into session.update", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+      instructions: "You are a helpful assistant.",
+      modalities: ["audio", "text"],
+      temperature: 0.7,
+      inputTranscription: { model: "whisper-1" },
+      toolChoice: "required",
+      inputRateHz: 16000,
+      outputRateHz: 16000,
+    });
+
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    expect(JSON.parse(mock.sent[0]!)).toEqual({
+      type: "session.update",
+      session: {
+        type: "realtime",
+        model: "gpt-realtime-2",
+        output_modalities: ["audio", "text"],
+        instructions: "You are a helpful assistant.",
+        temperature: 0.7,
+        audio: {
+          input: {
+            format: { type: "audio/pcm", rate: 16000 },
+            turn_detection: { type: "semantic_vad" },
+            transcription: { model: "whisper-1" },
+          },
+          output: {
+            format: { type: "audio/pcm", rate: 16000 },
+            voice: "marin",
+          },
+        },
+        tools: [],
+        tool_choice: "required",
+      },
+    });
+    expect(adapter.caps.inputSampleRateHz).toBe(16000);
+    expect(adapter.caps.outputSampleRateHz).toBe(16000);
+  });
+
+  it("R-10: skips response.create when requiresResponseCreateAfterToolOutput is false", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+      requiresResponseCreateAfterToolOutput: false,
+    });
+
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    adapter.injectToolResult("call_xyz", "done");
+    expect(mock.sent).toHaveLength(2);
+    expect(JSON.parse(mock.sent[1]!)).toEqual({
+      type: "conversation.item.create",
+      item: {
+        type: "function_call_output",
+        call_id: "call_xyz",
+        output: "done",
+      },
+    });
+  });
+
+  it("B1: does not send response.create while response is active (cancel-in-flight interleaving)", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+    });
+
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    mock.inject({ type: "response.created" });
+    mock.inject({
+      type: "response.output_item.added",
+      item: { type: "message", id: "item_assistant_1" },
+    });
+
+    adapter.cancelResponse(420);
+    expect(JSON.parse(mock.sent[1]!)).toEqual({ type: "response.cancel" });
+    expect(JSON.parse(mock.sent[2]!)).toEqual({
+      type: "conversation.item.truncate",
+      item_id: "item_assistant_1",
+      content_index: 0,
+      audio_end_ms: 420,
+    });
+
+    adapter.injectToolResult("call_abc", "Late Add Petition required.");
+    const typesAfterInject = mock.sent.map(
+      (raw) => (JSON.parse(raw) as Record<string, unknown>)["type"],
+    );
+    expect(typesAfterInject).toContain("conversation.item.create");
+    expect(typesAfterInject).not.toContain("response.create");
+
+    mock.inject({ type: "response.done", response: {} });
+    const typesAfterDone = mock.sent.map(
+      (raw) => (JSON.parse(raw) as Record<string, unknown>)["type"],
+    );
+    expect(typesAfterDone.filter((t) => t === "response.create")).toHaveLength(1);
+    expect(JSON.parse(mock.sent[mock.sent.length - 1]!)).toEqual({ type: "response.create" });
+  });
+
+  it("B1: does not send response.create while response is active (direct inject without cancel)", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+    });
+
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    mock.inject({ type: "response.created" });
+
+    adapter.injectToolResult("call_xyz", "inline result");
+    const typesAfterInject = mock.sent.map(
+      (raw) => (JSON.parse(raw) as Record<string, unknown>)["type"],
+    );
+    expect(typesAfterInject).toContain("conversation.item.create");
+    expect(typesAfterInject).not.toContain("response.create");
+
+    mock.inject({ type: "response.done", response: {} });
+    expect(
+      mock.sent.map((raw) => (JSON.parse(raw) as Record<string, unknown>)["type"]),
+    ).toContain("response.create");
+  });
+
+  it("R-11: does not truncate stale item when new response is canceled before output_item.added", async () => {
+    const mock = createMockSocketHarness();
+    const adapter = fromOpenAIRealtime({
+      apiKey: "test-key",
+      socketFactory: mock.factory,
+      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
+    });
+
+    const openTask = adapter.open(new AbortController().signal);
+    await waitFor(() => mock.sent.length > 0);
+    mock.inject({ type: "session.updated" });
+    await openTask;
+
+    mock.inject({ type: "response.created" });
+    mock.inject({
+      type: "response.output_item.added",
+      item: { type: "message", id: "item_old" },
+    });
+    mock.inject({ type: "response.done" });
+
+    mock.inject({ type: "response.created" });
+    adapter.cancelResponse(100);
+
+    const truncateMessages = mock.sent
+      .map((raw) => JSON.parse(raw) as Record<string, unknown>)
+      .filter((msg) => msg["type"] === "conversation.item.truncate");
+    expect(truncateMessages).toHaveLength(0);
+  });
+});

package/src/from-openai-realtime.ts

@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: MIT
+
+import type { SocketFactory } from "@kuralle-syrinx/ws";
+
+import { base64ToBytes, bytesToBase64 } from "./base64.js";
+import { createOpenAiCompatibleRealtimeAdapter } from "./openai-compatible-realtime.js";
+import type { RealtimeAdapter, RealtimeToolDef } from "./realtime-adapter.js";
+
+const DEFAULT_MODEL = "gpt-realtime-2";
+const DEFAULT_VOICE = "marin";
+const DEFAULT_SAMPLE_RATE_HZ = 24_000;
+
+export interface OpenAIRealtimeOptions {
+  readonly apiKey: string;
+  readonly model?: string;
+  readonly voice?: string;
+  readonly socketFactory: SocketFactory;
+  readonly url?: () => string;
+  /** Server turn-detection config. Defaults to semantic_vad; pass `server_vad` or `null` to override. */
+  readonly turnDetection?: Record<string, unknown> | null;
+  /**
+   * Function tools the front model may call (e.g. a delegate tool routed to a Reasoner). Domain-neutral
+   * — the caller supplies these. Empty by default (standalone s2s, no delegation).
+   */
+  readonly tools?: readonly RealtimeToolDef[];
+  readonly debug?: boolean;
+  /** gpt-realtime-2 requires response.create after function_call_output; set false for providers that do not. */
+  readonly requiresResponseCreateAfterToolOutput?: boolean;
+  readonly instructions?: string;
+  readonly modalities?: readonly string[];
+  readonly temperature?: number;
+  readonly inputTranscription?: Record<string, unknown> | boolean;
+  readonly toolChoice?: string | Record<string, unknown>;
+  readonly inputRateHz?: number;
+  readonly outputRateHz?: number;
+}
+
+export function fromOpenAIRealtime(opts: OpenAIRealtimeOptions): RealtimeAdapter {
+  const inputRateHz = opts.inputRateHz ?? DEFAULT_SAMPLE_RATE_HZ;
+  const outputRateHz = opts.outputRateHz ?? DEFAULT_SAMPLE_RATE_HZ;
+  const model = opts.model ?? DEFAULT_MODEL;
+  const voice = opts.voice ?? DEFAULT_VOICE;
+
+  return createOpenAiCompatibleRealtimeAdapter({
+    apiKey: opts.apiKey,
+    socketFactory: opts.socketFactory,
+    debug: opts.debug,
+    defaultModel: DEFAULT_MODEL,
+    model: opts.model,
+    url: opts.url,
+    caps: {
+      inputSampleRateHz: inputRateHz,
+      outputSampleRateHz: outputRateHz,
+      supportsConcurrentToolAudio: true,
+      supportsTruncate: true,
+      emitsServerSpeechStarted: true,
+    },
+    buildSessionUpdate: () => {
+      const inputAudio: Record<string, unknown> = {
+        format: { type: "audio/pcm", rate: inputRateHz },
+        turn_detection:
+          "turnDetection" in opts ? opts.turnDetection : { type: "semantic_vad" },
+      };
+      if (opts.inputTranscription !== undefined) {
+        inputAudio["transcription"] =
+          opts.inputTranscription === true ? { model: "whisper-1" } : opts.inputTranscription;
+      }
+
+      const session: Record<string, unknown> = {
+        type: "realtime",
+        model,
+        output_modalities: opts.modalities ?? ["audio"],
+        audio: {
+          input: inputAudio,
+          output: {
+            format: { type: "audio/pcm", rate: outputRateHz },
+            voice,
+          },
+        },
+        tools: (opts.tools ?? []).map((t) => ({
+          type: "function" as const,
+          name: t.name,
+          description: t.description,
+          parameters: t.parameters,
+        })),
+        tool_choice: opts.toolChoice ?? "auto",
+      };
+
+      if (opts.instructions !== undefined) {
+        session["instructions"] = opts.instructions;
+      }
+      if (opts.temperature !== undefined) {
+        session["temperature"] = opts.temperature;
+      }
+
+      return session;
+    },
+    supportsTruncate: true,
+    requiresResponseCreateAfterToolOutput: opts.requiresResponseCreateAfterToolOutput,
+    defaultErrorMessage: "OpenAI Realtime error",
+  });
+}
+
+export { bytesToBase64, base64ToBytes };

package/src/gemini-translate.test.ts

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: MIT
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { createGeminiTranslateSession } from "./gemini-translate.js";
+
+const sendRealtimeInput = vi.fn();
+const closeSession = vi.fn();
+
+const liveConnect = vi.fn().mockImplementation(async () => ({
+  sendRealtimeInput,
+  close: closeSession,
+}));
+
+vi.mock("@google/genai", () => ({
+  GoogleGenAI: vi.fn().mockImplementation(() => ({
+    live: { connect: liveConnect },
+  })),
+  Modality: { AUDIO: "AUDIO" },
+}));
+
+const FRAME_BYTES = 640;
+
+function frame(byte: number): Uint8Array {
+  return new Uint8Array(FRAME_BYTES).fill(byte);
+}
+
+afterEach(() => {
+  sendRealtimeInput.mockClear();
+  closeSession.mockClear();
+  liveConnect.mockClear();
+});
+
+describe("createGeminiTranslateSession", () => {
+  it("coalesces 20ms frames into 100ms chunks before sendRealtimeInput", async () => {
+    const session = await createGeminiTranslateSession({
+      apiKey: "test-key",
+      targetLanguageCode: "es",
+      onAudio: () => {},
+    });
+
+    for (let i = 0; i < 5; i += 1) {
+      session.sendAudio(frame(i));
+    }
+
+    expect(sendRealtimeInput).toHaveBeenCalledTimes(1);
+    const arg = sendRealtimeInput.mock.calls[0]![0] as { audio: { data: string } };
+    const chunk = Buffer.from(arg.audio.data, "base64");
+    expect(chunk.byteLength).toBe(3200);
+    expect(chunk[0]).toBe(0);
+    expect(chunk[639]).toBe(0);
+    expect(chunk[640]).toBe(1);
+    expect(chunk[3199]).toBe(4);
+  });
+
+  it("flushes remainder on signalAudioStreamEnd and close", async () => {
+    const session = await createGeminiTranslateSession({
+      apiKey: "test-key",
+      targetLanguageCode: "es",
+      onAudio: () => {},
+    });
+
+    session.sendAudio(frame(9));
+    session.sendAudio(frame(9));
+    session.sendAudio(frame(9));
+    expect(sendRealtimeInput).toHaveBeenCalledTimes(0);
+
+    session.signalAudioStreamEnd();
+    expect(sendRealtimeInput).toHaveBeenCalledTimes(2);
+    const streamEndArg = sendRealtimeInput.mock.calls[0]![0] as { audio: { data: string } };
+    expect(Buffer.from(streamEndArg.audio.data, "base64").byteLength).toBe(1920);
+    expect(sendRealtimeInput.mock.calls[1]![0]).toEqual({ audioStreamEnd: true });
+
+    sendRealtimeInput.mockClear();
+    session.sendAudio(frame(7));
+    await session.close();
+    expect(sendRealtimeInput).toHaveBeenCalledTimes(1);
+    const closeArg = sendRealtimeInput.mock.calls[0]![0] as { audio: { data: string } };
+    expect(Buffer.from(closeArg.audio.data, "base64").byteLength).toBe(640);
+    expect(closeSession).toHaveBeenCalledTimes(1);
+  });
+});