@kuralle-syrinx/realtime 2.1.0

package/src/gemini-translate.ts

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: MIT
+
+import type { Session, LiveServerMessage } from "@google/genai";
+
+import { bytesToBase64, base64ToBytes } from "./base64.js";
+
+const DEFAULT_MODEL = "gemini-3.5-live-translate-preview";
+const OUTPUT_SAMPLE_RATE_HZ = 24_000;
+
+export interface GeminiTranslateSession {
+  sendAudio(pcm16: Uint8Array): void;
+  signalAudioStreamEnd(): void;
+  close(): Promise<void>;
+}
+
+export interface GeminiTranslateSessionOptions {
+  readonly apiKey: string;
+  readonly targetLanguageCode: string;
+  readonly echoTargetLanguage?: boolean;
+  readonly onAudio: (pcm16: Uint8Array, sampleRateHz: number) => void;
+  readonly onText?: (text: string, role: "input" | "output", final: boolean) => void;
+  readonly onError?: (cause: Error) => void;
+}
+
+export async function createGeminiTranslateSession(
+  opts: GeminiTranslateSessionOptions,
+): Promise<GeminiTranslateSession> {
+  const { GoogleGenAI, Modality } = await import("@google/genai");
+  const ai = new GoogleGenAI({
+    apiKey: opts.apiKey,
+    httpOptions: { apiVersion: "v1alpha" },
+  });
+
+  const session = await ai.live.connect({
+    model: DEFAULT_MODEL,
+    config: {
+      responseModalities: [Modality.AUDIO],
+      inputAudioTranscription: {},
+      outputAudioTranscription: {},
+      translationConfig: {
+        targetLanguageCode: opts.targetLanguageCode,
+        echoTargetLanguage: opts.echoTargetLanguage ?? true,
+      },
+    },
+    callbacks: {
+      onmessage: (msg) => handleTranslateMessage(msg, opts),
+      onerror: (ev) => {
+        const cause = ev instanceof Error ? ev : new Error(String(ev));
+        opts.onError?.(cause);
+      },
+    },
+  });
+
+  // Gemini Live Translate needs ~100ms input chunks. Feeding the engine's native 20ms frames
+  // intermittently makes the model echo the SOURCE language instead of translating (~20-33%,
+  // verified deterministically via Deepgram detect_language). Coalesce to 100ms here so callers
+  // can keep sending 20ms frames. 100ms @ 16kHz PCM16 = 1600 samples = 3200 bytes (sample-aligned).
+  const INPUT_CHUNK_BYTES = 3200;
+  let buf = new Uint8Array(0);
+  const flushChunk = (chunk: Uint8Array): void => {
+    session.sendRealtimeInput({ audio: { data: bytesToBase64(chunk), mimeType: "audio/pcm;rate=16000" } });
+  };
+  const flushRemainder = (): void => {
+    if (buf.length > 0) {
+      flushChunk(buf);
+      buf = new Uint8Array(0);
+    }
+  };
+
+  return {
+    sendAudio(pcm16: Uint8Array): void {
+      const merged = new Uint8Array(buf.length + pcm16.length);
+      merged.set(buf, 0);
+      merged.set(pcm16, buf.length);
+      buf = merged;
+      while (buf.length >= INPUT_CHUNK_BYTES) {
+        flushChunk(buf.subarray(0, INPUT_CHUNK_BYTES));
+        buf = buf.slice(INPUT_CHUNK_BYTES);
+      }
+    },
+    signalAudioStreamEnd(): void {
+      flushRemainder();
+      session.sendRealtimeInput({ audioStreamEnd: true });
+    },
+    async close(): Promise<void> {
+      flushRemainder();
+      session.close();
+    },
+  };
+}
+
+function handleTranslateMessage(
+  msg: LiveServerMessage,
+  opts: GeminiTranslateSessionOptions,
+): void {
+  const content = msg.serverContent;
+  if (!content) return;
+
+  if (content.inputTranscription?.text && opts.onText) {
+    opts.onText(
+      content.inputTranscription.text,
+      "input",
+      content.inputTranscription.finished ?? false,
+    );
+  }
+
+  if (content.outputTranscription?.text && opts.onText) {
+    opts.onText(
+      content.outputTranscription.text,
+      "output",
+      content.outputTranscription.finished ?? false,
+    );
+  }
+
+  const parts = content.modelTurn?.parts;
+  if (!parts) return;
+
+  for (const part of parts) {
+    const inline = part.inlineData;
+    if (inline?.data && inline.mimeType?.startsWith("audio/")) {
+      const rateMatch = /rate=(\d+)/.exec(inline.mimeType);
+      const sampleRateHz = rateMatch ? Number(rateMatch[1]) : OUTPUT_SAMPLE_RATE_HZ;
+      opts.onAudio(base64ToBytes(inline.data), sampleRateHz);
+    }
+  }
+}
+
+export { DEFAULT_MODEL as GEMINI_TRANSLATE_MODEL };

package/src/index.ts

@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: MIT
+
+export type { RealtimeAdapter, RealtimeEvent, RealtimeToolDef } from "./realtime-adapter.js";
+export { bytesToBase64, base64ToBytes } from "./base64.js";
+export {
+  createOpenAiCompatibleRealtimeAdapter,
+  type OpenAiCompatibleRealtimeConfig,
+} from "./openai-compatible-realtime.js";
+export { fromOpenAIRealtime, type OpenAIRealtimeOptions } from "./from-openai-realtime.js";
+export { fromGeminiLive, type GeminiLiveOptions } from "./from-gemini-live.js";
+export {
+  createGeminiTranslateSession,
+  GEMINI_TRANSLATE_MODEL,
+  type GeminiTranslateSession,
+  type GeminiTranslateSessionOptions,
+} from "./gemini-translate.js";
+export { RealtimeBridge } from "./realtime-bridge.js";

package/src/openai-compatible-realtime.ts

@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: MIT
+
+import type { SocketFactory } from "@kuralle-syrinx/ws";
+import { RealtimeSocket } from "@kuralle-syrinx/ws/realtime";
+
+import { base64ToBytes, bytesToBase64 } from "./base64.js";
+import type { RealtimeAdapter, RealtimeEvent } from "./realtime-adapter.js";
+
+export interface OpenAiCompatibleRealtimeConfig {
+  readonly apiKey: string;
+  readonly socketFactory: SocketFactory;
+  readonly debug?: boolean;
+  readonly debugLogPrefix?: string;
+  readonly defaultModel: string;
+  readonly model?: string;
+  readonly url?: () => string;
+  readonly buildUrl?: (model: string) => string;
+  readonly caps: RealtimeAdapter["caps"];
+  readonly buildSessionUpdate: () => Record<string, unknown>;
+  readonly supportsTruncate: boolean;
+  readonly requiresResponseCreateAfterToolOutput?: boolean;
+  readonly defaultErrorMessage: string;
+  readonly extendServerMessage?: (
+    type: string,
+    msg: Record<string, unknown>,
+    ctx: {
+      push: (event: RealtimeEvent) => void;
+      caps: RealtimeAdapter["caps"];
+    },
+  ) => boolean;
+}
+
+class RealtimeEventStream implements AsyncIterable<RealtimeEvent> {
+  private readonly queue: RealtimeEvent[] = [];
+  private readonly waiters: Array<(result: IteratorResult<RealtimeEvent>) => void> = [];
+  private closed = false;
+
+  push(event: RealtimeEvent): void {
+    if (this.closed) return;
+    const waiter = this.waiters.shift();
+    if (waiter) {
+      waiter({ value: event, done: false });
+      return;
+    }
+    this.queue.push(event);
+  }
+
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    while (this.waiters.length > 0) {
+      const waiter = this.waiters.shift();
+      waiter?.({ value: undefined, done: true });
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<RealtimeEvent> {
+    return {
+      next: () =>
+        new Promise<IteratorResult<RealtimeEvent>>((resolve) => {
+          if (this.queue.length > 0) {
+            resolve({ value: this.queue.shift()!, done: false });
+            return;
+          }
+          if (this.closed) {
+            resolve({ value: undefined, done: true });
+            return;
+          }
+          this.waiters.push(resolve);
+        }),
+    };
+  }
+}
+
+class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
+  readonly caps: RealtimeAdapter["caps"];
+  readonly events: AsyncIterable<RealtimeEvent>;
+
+  private readonly stream = new RealtimeEventStream();
+  private socket: RealtimeSocket | null = null;
+  private abortHandler: (() => void) | null = null;
+  private openResolver: (() => void) | null = null;
+  private openRejecter: ((err: Error) => void) | null = null;
+  private currentAssistantItemId: string | null = null;
+  private assistantTranscript = "";
+  private activeResponse = false;
+  private pendingResponseCreate = false;
+
+  constructor(private readonly config: OpenAiCompatibleRealtimeConfig) {
+    this.events = this.stream;
+    this.caps = config.caps;
+  }
+
+  async open(signal: AbortSignal): Promise<void> {
+    const model = this.config.model ?? this.config.defaultModel;
+    const url =
+      this.config.url ??
+      (this.config.buildUrl
+        ? () => this.config.buildUrl!(model)
+        : () => `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`);
+
+    this.socket = new RealtimeSocket({
+      url,
+      headers: {
+        Authorization: `Bearer ${this.config.apiKey}`,
+      },
+      socketFactory: this.config.socketFactory,
+      onMessage: (json) => this.handleServerMessage(json),
+      onConnectionLost: (err) => {
+        this.stream.push({ type: "error", cause: err, recoverable: true });
+        this.rejectOpen(err);
+      },
+      onUnrecoverable: (err) => {
+        this.stream.push({ type: "error", cause: err, recoverable: false });
+        this.rejectOpen(err);
+      },
+    });
+
+    const openPromise = new Promise<void>((resolve, reject) => {
+      this.openResolver = resolve;
+      this.openRejecter = reject;
+    });
+
+    this.abortHandler = () => {
+      void this.close();
+      this.rejectOpen(new Error("Realtime adapter open aborted"));
+    };
+    signal.addEventListener("abort", this.abortHandler, { once: true });
+
+    await this.socket.connect();
+    this.socket.send({
+      type: "session.update",
+      session: this.config.buildSessionUpdate(),
+    });
+
+    await openPromise;
+  }
+
+  sendAudio(pcm16: Uint8Array): void {
+    this.requireSocket().send({
+      type: "input_audio_buffer.append",
+      audio: bytesToBase64(pcm16),
+    });
+  }
+
+  cancelResponse(audioEndMs: number): void {
+    if (!this.activeResponse) return;
+    const socket = this.requireSocket();
+    socket.send({ type: "response.cancel" });
+    if (this.config.supportsTruncate && this.currentAssistantItemId) {
+      socket.send({
+        type: "conversation.item.truncate",
+        item_id: this.currentAssistantItemId,
+        content_index: 0,
+        audio_end_ms: audioEndMs,
+      });
+    }
+    if (this.config.supportsTruncate) {
+      this.currentAssistantItemId = null;
+    }
+  }
+
+  injectToolResult(toolId: string, text: string): void {
+    const socket = this.requireSocket();
+    socket.send({
+      type: "conversation.item.create",
+      item: {
+        type: "function_call_output",
+        call_id: toolId,
+        output: text,
+      },
+    });
+    if (this.config.requiresResponseCreateAfterToolOutput !== false) {
+      this.requestResponseCreate();
+    }
+  }
+
+  async close(): Promise<void> {
+    await this.socket?.close();
+    this.socket = null;
+    this.stream.close();
+  }
+
+  private requestResponseCreate(): void {
+    if (this.activeResponse) {
+      this.pendingResponseCreate = true;
+      return;
+    }
+    this.requireSocket().send({ type: "response.create" });
+    this.pendingResponseCreate = false;
+  }
+
+  private completeResponse(): void {
+    this.activeResponse = false;
+    if (this.config.supportsTruncate) {
+      this.currentAssistantItemId = null;
+    }
+    if (this.pendingResponseCreate) {
+      this.pendingResponseCreate = false;
+      this.requireSocket().send({ type: "response.create" });
+    }
+  }
+
+  private requireSocket(): RealtimeSocket {
+    if (!this.socket) throw new Error("Realtime adapter is not open");
+    return this.socket;
+  }
+
+  private rejectOpen(err: Error): void {
+    this.openRejecter?.(err);
+    this.openResolver = null;
+    this.openRejecter = null;
+  }
+
+  private resolveOpen(): void {
+    this.openResolver?.();
+    this.openResolver = null;
+    this.openRejecter = null;
+  }
+
+  private handleServerMessage(json: string): void {
+    let msg: Record<string, unknown>;
+    try {
+      msg = JSON.parse(json) as Record<string, unknown>;
+    } catch (err) {
+      this.stream.push({
+        type: "error",
+        cause: err instanceof Error ? err : new Error(String(err)),
+        recoverable: false,
+      });
+      return;
+    }
+
+    const type = typeof msg["type"] === "string" ? msg["type"] : "";
+    if (this.config.debug) {
+      console.error(`${this.config.debugLogPrefix ?? "[raw]"} ${type}`);
+    }
+
+    if (this.config.extendServerMessage?.(type, msg, { push: (e) => this.stream.push(e), caps: this.caps })) {
+      return;
+    }
+
+    switch (type) {
+      case "session.created":
+      case "session.updated":
+        this.resolveOpen();
+        break;
+      case "response.created":
+        this.assistantTranscript = "";
+        if (this.config.supportsTruncate) {
+          this.currentAssistantItemId = null;
+        }
+        this.activeResponse = true;
+        this.stream.push({ type: "response_started" });
+        break;
+      case "response.output_item.added": {
+        if (!this.config.supportsTruncate) break;
+        const item = msg["item"];
+        if (item && typeof item === "object") {
+          const record = item as Record<string, unknown>;
+          if (record["type"] === "message" && typeof record["id"] === "string") {
+            this.currentAssistantItemId = record["id"];
+          }
+        }
+        break;
+      }
+      case "response.output_audio.delta": {
+        const delta = msg["delta"];
+        if (typeof delta === "string" && delta.length > 0) {
+          this.stream.push({
+            type: "audio",
+            pcm16: base64ToBytes(delta),
+            sampleRateHz: this.caps.outputSampleRateHz,
+          });
+        }
+        break;
+      }
+      case "input_audio_buffer.speech_started":
+        this.stream.push({ type: "speech_started" });
+        break;
+      case "response.output_audio_transcript.delta": {
+        const delta = msg["delta"];
+        if (typeof delta === "string" && delta.length > 0) {
+          this.assistantTranscript += delta;
+          this.stream.push({
+            type: "transcript",
+            role: "assistant",
+            text: delta,
+            final: false,
+          });
+        }
+        break;
+      }
+      case "response.output_audio_transcript.done": {
+        const transcript =
+          typeof msg["transcript"] === "string" ? msg["transcript"] : this.assistantTranscript;
+        this.stream.push({
+          type: "transcript",
+          role: "assistant",
+          text: transcript,
+          final: true,
+        });
+        this.assistantTranscript = "";
+        break;
+      }
+      case "conversation.item.input_audio_transcription.completed": {
+        // User-side transcript (requires input transcription enabled in session config).
+        const transcript = typeof msg["transcript"] === "string" ? msg["transcript"] : "";
+        if (transcript.trim()) {
+          this.stream.push({ type: "transcript", role: "user", text: transcript, final: true });
+        }
+        break;
+      }
+      case "response.done": {
+        this.completeResponse();
+        const toolCall = extractFunctionCall(msg["response"]);
+        if (toolCall) {
+          this.stream.push(toolCall);
+        }
+        this.stream.push({ type: "response_done" });
+        break;
+      }
+      case "error": {
+        const errObj = msg["error"];
+        const message =
+          errObj && typeof errObj === "object" && typeof (errObj as Record<string, unknown>)["message"] === "string"
+            ? String((errObj as Record<string, unknown>)["message"])
+            : this.config.defaultErrorMessage;
+        const code =
+          errObj && typeof errObj === "object" ? (errObj as Record<string, unknown>)["code"] : undefined;
+        const recoverable = code !== "invalid_api_key" && code !== "authentication_failed";
+        this.stream.push({
+          type: "error",
+          cause: new Error(message),
+          recoverable,
+        });
+        break;
+      }
+      default:
+        break;
+    }
+  }
+}
+
+function extractFunctionCall(response: unknown): RealtimeEvent | null {
+  if (!response || typeof response !== "object") return null;
+  const output = (response as Record<string, unknown>)["output"];
+  if (!Array.isArray(output)) return null;
+  for (const item of output) {
+    if (!item || typeof item !== "object") continue;
+    const record = item as Record<string, unknown>;
+    if (record["type"] !== "function_call") continue;
+    const toolId = typeof record["call_id"] === "string" ? record["call_id"] : "";
+    const toolName = typeof record["name"] === "string" ? record["name"] : "";
+    const argsRaw = typeof record["arguments"] === "string" ? record["arguments"] : "{}";
+    if (!toolId || !toolName) continue;
+    let args: Record<string, unknown>;
+    try {
+      const parsed: unknown = JSON.parse(argsRaw);
+      args = parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : {};
+    } catch {
+      args = {};
+    }
+    return { type: "tool_call", toolId, toolName, args };
+  }
+  return null;
+}
+
+export function createOpenAiCompatibleRealtimeAdapter(
+  config: OpenAiCompatibleRealtimeConfig,
+): RealtimeAdapter {
+  return new OpenAiCompatibleRealtimeAdapter(config);
+}

package/src/realtime-adapter.ts

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: MIT
+
+export interface RealtimeAdapter {
+  readonly caps: {
+    readonly inputSampleRateHz: number;
+    readonly outputSampleRateHz: number;
+    readonly supportsConcurrentToolAudio: boolean;
+    readonly supportsTruncate: boolean;
+    readonly emitsServerSpeechStarted: boolean;
+  };
+
+  open(signal: AbortSignal): Promise<void>;
+  sendAudio(pcm16: Uint8Array): void;
+  cancelResponse(audioEndMs: number): void;
+  injectToolResult(toolId: string, text: string): void;
+  /** Close the provider socket and end the event stream. */
+  close(): Promise<void>;
+  readonly events: AsyncIterable<RealtimeEvent>;
+}
+
+/**
+ * A function tool advertised to the front model so it can decide when to delegate. Domain-neutral:
+ * the caller (example/app) supplies these — the provider adapter never hardcodes any tool.
+ */
+export interface RealtimeToolDef {
+  readonly name: string;
+  readonly description: string;
+  /** JSON Schema for the tool arguments. */
+  readonly parameters: Record<string, unknown>;
+}
+
+export type RealtimeEvent =
+  | { type: "audio"; pcm16: Uint8Array; sampleRateHz: number }
+  | { type: "speech_started" }
+  | { type: "transcript"; role: "user" | "assistant"; text: string; final: boolean }
+  | { type: "tool_call"; toolId: string; toolName: string; args: Record<string, unknown> }
+  | { type: "response_started" }
+  | { type: "response_done" }
+  | { type: "error"; cause: Error; recoverable: boolean };