@kuralle-syrinx/realtime 4.2.0 → 4.4.0

package/README.md

@@ -105,6 +105,34 @@ Cartesia, …) all dial through `createWorkersSocket` so auth headers ride on th
 Regression lock: `edge-safety.test.ts` runs the adapter + bridge with `Buffer` and `process` removed from
 `globalThis`.
 
+### Gemini Live transcription, voice, and API version
+
+`fromGeminiLive` keeps both transcription directions enabled by default and makes each one
+configurable (set either to `false` to disable it):
+
+```ts
+const adapter = fromGeminiLive({
+  apiKey,
+  transcription: { input: true, output: true },
+  speechConfig: { voice: "Kore", languageCode: "en-US" },
+  apiVersion: "v1alpha",
+});
+```
+
+The adapter maps these options to Gemini Live's `inputAudioTranscription`,
+`outputAudioTranscription`, and `speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName` setup fields.
+The server reports both transcripts in `serverContent.inputTranscription` and
+`serverContent.outputTranscription`, which become `{ type: "transcript", role, text, final }` events.
+
+Google's [Live API capabilities guide](https://ai.google.dev/gemini-api/docs/live-api/capabilities)
+currently documents input transcription as unsupported by Gemini 3.1 Flash Live Preview. When using
+that model, enabling `transcription.input` will not produce user transcripts; use a Live model that
+supports input transcription or a separate STT provider when user transcripts are required. Native
+audio models also choose their output language automatically, so `languageCode` is intended for models
+that support explicit language selection. See Google's [API version guide](https://ai.google.dev/gemini-api/docs/api-versions)
+for version semantics; `apiVersion` is applied at SDK client level because Live does not support
+request-level HTTP options.
+
 ## The Responder-Thinker primitive (the delegate seam)
 
 The bi-model shape has a name: **Responder-Thinker** — a fast realtime **responder** on the audio

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/realtime",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "private": false,
   "description": "Realtime speech-to-speech bridge for Syrinx — OpenAI Realtime, Gemini Live, and openai-compatible adapters; the Responder-Thinker delegate seam",
   "keywords": [
@@ -28,12 +28,12 @@
   "types": "./src/index.ts",
   "dependencies": {
     "@google/genai": "^2.8.0",
-    "@kuralle-syrinx/core": "4.2.0",
-    "@kuralle-syrinx/ws": "4.2.0"
+    "@kuralle-syrinx/ws": "4.4.0",
+    "@kuralle-syrinx/core": "4.4.0"
   },
   "devDependencies": {
     "typescript": "^5.7.0",
-    "vitest": "^2.1.0"
+    "vitest": "^3.2.6"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",

package/src/from-gemini-live.ts

@@ -10,11 +10,35 @@ const DEFAULT_MODEL = "gemini-3.1-flash-live-preview";
 const INPUT_SAMPLE_RATE_HZ = 16_000;
 const OUTPUT_SAMPLE_RATE_HZ = 24_000;
 
+export interface GeminiLiveTranscriptionOptions {
+  /**
+   * Enable input transcription, or provide Gemini's AudioTranscriptionConfig. Enabled by default.
+   *
+   * Default stays ON because `RealtimeBridge` turns `role: "user"` transcripts into `stt.result`
+   * packets — defaulting this off would silently remove all user-side text on the Gemini front.
+   * Issue #32 asked for it to be *configurable*, not disabled.
+   */
+  readonly input?: boolean | Record<string, unknown>;
+  /** Enable output transcription, or provide Gemini's AudioTranscriptionConfig. Enabled by default. */
+  readonly output?: boolean | Record<string, unknown>;
+}
+
+export interface GeminiLiveSpeechConfig {
+  /** Prebuilt Gemini voice name, mapped to speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName. */
+  readonly voice?: string;
+  /** ISO language code for speech synthesis. */
+  readonly languageCode?: string;
+}
+
 export interface GeminiLiveOptions {
   readonly apiKey: string;
   readonly model?: string;
   readonly systemInstruction?: string;
   readonly tools?: readonly RealtimeToolDef[];
+  readonly transcription?: GeminiLiveTranscriptionOptions;
+  readonly speechConfig?: GeminiLiveSpeechConfig;
+  /** Gemini Developer API version, e.g. `v1alpha` for preview-only Live features. */
+  readonly apiVersion?: string;
   /**
    * G4 native resume: a `sessionResumption` handle from a prior session's
    * `resumption_handle` events. Gemini restores the conversation server-side —
@@ -23,6 +47,34 @@ export interface GeminiLiveOptions {
    * prior handle back on reconnect.
    */
   readonly sessionResumptionHandle?: string;
+  /**
+   * Full `sessionResumption` LiveConnectConfig. Defaults to `{}` (handles issued)
+   * or `{ handle }` when `sessionResumptionHandle` is set. Pass `false` to omit.
+   */
+  readonly sessionResumption?: Record<string, unknown> | false;
+  /** Response modalities. Defaults to `["AUDIO"]` (previous hard-pin). */
+  readonly responseModalities?: readonly string[];
+  /** LiveConnectConfig `generationConfig` (subset supported by Live). */
+  readonly generationConfig?: Record<string, unknown>;
+  /** LiveConnectConfig `safetySettings`. */
+  readonly safetySettings?: readonly Record<string, unknown>[];
+  readonly temperature?: number;
+  readonly topP?: number;
+  readonly topK?: number;
+  readonly maxOutputTokens?: number;
+  readonly mediaResolution?: string;
+  readonly seed?: number;
+  readonly thinkingConfig?: Record<string, unknown>;
+  readonly enableAffectiveDialog?: boolean;
+  readonly realtimeInputConfig?: Record<string, unknown>;
+  readonly contextWindowCompression?: Record<string, unknown>;
+  readonly proactivity?: Record<string, unknown>;
+  readonly explicitVadSignal?: boolean;
+  /**
+   * Merged last into `live.connect` config for any LiveConnectConfig field the
+   * adapter does not enumerate (avatarConfig, …). Overrides same-key defaults.
+   */
+  readonly connectConfig?: Record<string, unknown>;
 }
 
 class GeminiLiveAdapter implements RealtimeAdapter {
@@ -61,29 +113,106 @@ class GeminiLiveAdapter implements RealtimeAdapter {
       }],
     }));
 
+    const transcription = this.opts.transcription;
     const config: Record<string, unknown> = {
-      responseModalities: [Modality.AUDIO],
-      inputAudioTranscription: {},
-      outputAudioTranscription: {},
-      // G4: always on so the server issues resumption handles; a prior handle
-      // resumes the conversation server-side (native resume — no replay).
-      sessionResumption: this.opts.sessionResumptionHandle
-        ? { handle: this.opts.sessionResumptionHandle }
-        : {},
+      // Default preserves the prior hard-pin of AUDIO-only responses.
+      responseModalities: this.opts.responseModalities
+        ? [...this.opts.responseModalities]
+        : [Modality.AUDIO],
     };
+    // G4: session resumption defaults ON so the server issues handles; a prior
+    // handle resumes server-side (native resume — no replay). Override via
+    // sessionResumption (full object) or disable with false.
+    if (!("sessionResumption" in this.opts) || this.opts.sessionResumption !== false) {
+      const base =
+        typeof this.opts.sessionResumption === "object" && this.opts.sessionResumption !== null
+          ? { ...this.opts.sessionResumption }
+          : {};
+      if (this.opts.sessionResumptionHandle !== undefined) {
+        base["handle"] = this.opts.sessionResumptionHandle;
+      }
+      config["sessionResumption"] = base;
+    }
+    const inputTranscription = transcription?.input ?? true;
+    const outputTranscription = transcription?.output ?? true;
+    if (inputTranscription !== false) {
+      config["inputAudioTranscription"] = inputTranscription === true ? {} : inputTranscription;
+    }
+    if (outputTranscription !== false) {
+      config["outputAudioTranscription"] = outputTranscription === true ? {} : outputTranscription;
+    }
+    const speechConfig = this.opts.speechConfig;
+    if (speechConfig && (speechConfig.voice !== undefined || speechConfig.languageCode !== undefined)) {
+      config["speechConfig"] = {
+        ...(speechConfig.voice === undefined
+          ? {}
+          : { voiceConfig: { prebuiltVoiceConfig: { voiceName: speechConfig.voice } } }),
+        ...(speechConfig.languageCode === undefined ? {} : { languageCode: speechConfig.languageCode }),
+      };
+    }
     if (this.opts.systemInstruction) {
       config["systemInstruction"] = this.opts.systemInstruction;
     }
     if (tools.length > 0) {
       config["tools"] = tools;
     }
+    if (this.opts.generationConfig !== undefined) {
+      config["generationConfig"] = this.opts.generationConfig;
+    }
+    if (this.opts.safetySettings !== undefined) {
+      config["safetySettings"] = this.opts.safetySettings;
+    }
+    if (this.opts.temperature !== undefined) {
+      config["temperature"] = this.opts.temperature;
+    }
+    if (this.opts.topP !== undefined) {
+      config["topP"] = this.opts.topP;
+    }
+    if (this.opts.topK !== undefined) {
+      config["topK"] = this.opts.topK;
+    }
+    if (this.opts.maxOutputTokens !== undefined) {
+      config["maxOutputTokens"] = this.opts.maxOutputTokens;
+    }
+    if (this.opts.mediaResolution !== undefined) {
+      config["mediaResolution"] = this.opts.mediaResolution;
+    }
+    if (this.opts.seed !== undefined) {
+      config["seed"] = this.opts.seed;
+    }
+    if (this.opts.thinkingConfig !== undefined) {
+      config["thinkingConfig"] = this.opts.thinkingConfig;
+    }
+    if (this.opts.enableAffectiveDialog !== undefined) {
+      config["enableAffectiveDialog"] = this.opts.enableAffectiveDialog;
+    }
+    if (this.opts.realtimeInputConfig !== undefined) {
+      config["realtimeInputConfig"] = this.opts.realtimeInputConfig;
+    }
+    if (this.opts.contextWindowCompression !== undefined) {
+      config["contextWindowCompression"] = this.opts.contextWindowCompression;
+    }
+    if (this.opts.proactivity !== undefined) {
+      config["proactivity"] = this.opts.proactivity;
+    }
+    if (this.opts.explicitVadSignal !== undefined) {
+      config["explicitVadSignal"] = this.opts.explicitVadSignal;
+    }
+    if (this.opts.connectConfig) {
+      Object.assign(config, this.opts.connectConfig);
+    }
 
     const openPromise = new Promise<void>((resolve, reject) => {
       this.openResolver = resolve;
       this.openRejecter = reject;
     });
 
-    const ai = new GoogleGenAI({ apiKey: this.opts.apiKey });
+    const ai = new GoogleGenAI({
+      apiKey: this.opts.apiKey,
+      ...(this.opts.apiVersion === undefined
+        ? {}
+        : { httpOptions: { apiVersion: this.opts.apiVersion } }),
+    });
 
     this.session = await ai.live.connect({
       model,
@@ -125,6 +254,15 @@ class GeminiLiveAdapter implements RealtimeAdapter {
     });
   }
 
+  injectContext(text: string): void {
+    // Gemini Live drops system/developer roles from conversation history. A silent,
+    // incomplete user turn preserves the steering context without requesting a response.
+    this.requireSession().sendClientContent({
+      turns: [{ role: "user", parts: [{ text: `[Context-only instruction]\n${text}` }] }],
+      turnComplete: false,
+    });
+  }
+
   cancelResponse(_audioEndMs: number): void {
     // Gemini handles interruption server-side via `interrupted`; no truncate API.
   }

package/src/from-openai-realtime.ts

@@ -33,6 +33,19 @@ export interface OpenAIRealtimeOptions {
   readonly toolChoice?: string | Record<string, unknown>;
   readonly inputRateHz?: number;
   readonly outputRateHz?: number;
+  /** Input audio format object. Defaults to `{ type: "audio/pcm", rate: inputRateHz }`. */
+  readonly inputAudioFormat?: Record<string, unknown>;
+  /** Output audio format object. Defaults to `{ type: "audio/pcm", rate: outputRateHz }`. */
+  readonly outputAudioFormat?: Record<string, unknown>;
+  /** OpenAI Realtime `audio.input.noise_reduction` (e.g. `{ type: "near_field" }`). */
+  readonly noiseReduction?: Record<string, unknown> | null;
+  /** OpenAI Realtime `max_output_tokens` session field. */
+  readonly maxOutputTokens?: number | "inf";
+  /**
+   * Merged last into `session.update.session` for any provider field the adapter does not enumerate
+   * (speed, tracing, prompt, include, …). Overrides same-key defaults when both set.
+   */
+  readonly sessionConfig?: Record<string, unknown>;
   /**
    * G4 resume-by-replay: returns the prior conversation to replay as
    * `conversation.item.create` items on every (re)connect — OpenAI Realtime has no
@@ -64,7 +77,7 @@ export function fromOpenAIRealtime(opts: OpenAIRealtimeOptions): RealtimeAdapter
     },
     buildSessionUpdate: () => {
       const inputAudio: Record<string, unknown> = {
-        format: { type: "audio/pcm", rate: inputRateHz },
+        format: opts.inputAudioFormat ?? { type: "audio/pcm", rate: inputRateHz },
         turn_detection:
           "turnDetection" in opts ? opts.turnDetection : { type: "semantic_vad" },
       };
@@ -72,6 +85,14 @@ export function fromOpenAIRealtime(opts: OpenAIRealtimeOptions): RealtimeAdapter
         inputAudio["transcription"] =
           opts.inputTranscription === true ? { model: "whisper-1" } : opts.inputTranscription;
       }
+      if (opts.noiseReduction !== undefined) {
+        inputAudio["noise_reduction"] = opts.noiseReduction;
+      }
+
+      const outputAudio: Record<string, unknown> = {
+        format: opts.outputAudioFormat ?? { type: "audio/pcm", rate: outputRateHz },
+        voice,
+      };
 
       const session: Record<string, unknown> = {
         type: "realtime",
@@ -79,10 +100,7 @@ export function fromOpenAIRealtime(opts: OpenAIRealtimeOptions): RealtimeAdapter
         output_modalities: opts.modalities ?? ["audio"],
         audio: {
           input: inputAudio,
-          output: {
-            format: { type: "audio/pcm", rate: outputRateHz },
-            voice,
-          },
+          output: outputAudio,
         },
         tools: (opts.tools ?? []).map((t) => ({
           type: "function" as const,
@@ -99,6 +117,12 @@ export function fromOpenAIRealtime(opts: OpenAIRealtimeOptions): RealtimeAdapter
       if (opts.temperature !== undefined) {
         session["temperature"] = opts.temperature;
       }
+      if (opts.maxOutputTokens !== undefined) {
+        session["max_output_tokens"] = opts.maxOutputTokens;
+      }
+      if (opts.sessionConfig) {
+        Object.assign(session, opts.sessionConfig);
+      }
 
       return session;
     },

package/src/index.ts

@@ -12,7 +12,12 @@ export {
   type OpenAiCompatibleRealtimeConfig,
 } from "./openai-compatible-realtime.js";
 export { fromOpenAIRealtime, type OpenAIRealtimeOptions } from "./from-openai-realtime.js";
-export { fromGeminiLive, type GeminiLiveOptions } from "./from-gemini-live.js";
+export {
+  fromGeminiLive,
+  type GeminiLiveOptions,
+  type GeminiLiveSpeechConfig,
+  type GeminiLiveTranscriptionOptions,
+} from "./from-gemini-live.js";
 export {
   createGeminiTranslateSession,
   GEMINI_TRANSLATE_MODEL,

package/src/openai-compatible-realtime.ts

@@ -5,7 +5,7 @@ import { RealtimeSocket } from "@kuralle-syrinx/ws/realtime";
 
 import { base64ToBytes, bytesToBase64 } from "./base64.js";
 import { RealtimeEventStream } from "./realtime-event-stream.js";
-import type { RealtimeAdapter, RealtimeEvent, RealtimeResumeMessage } from "./realtime-adapter.js";
+import type { RealtimeAdapter, RealtimeEvent, RealtimeResumeMessage, RealtimeUsage } from "./realtime-adapter.js";
 
 export interface OpenAiCompatibleRealtimeConfig {
   readonly apiKey: string;
@@ -18,6 +18,11 @@ export interface OpenAiCompatibleRealtimeConfig {
   readonly buildUrl?: (model: string) => string;
   readonly caps: RealtimeAdapter["caps"];
   readonly buildSessionUpdate: () => Record<string, unknown>;
+  /**
+   * Optional fields merged into `session.update.session` after `buildSessionUpdate()`.
+   * Use for provider-specific knobs the adapter does not enumerate (mirrors TTS `extra_body`).
+   */
+  readonly sessionExtra?: Record<string, unknown> | (() => Record<string, unknown>);
   /**
    * G4 resume-by-replay: returns the prior conversation to re-create as
    * `conversation.item.create` items after each (re)connect's `session.update` —
@@ -117,9 +122,14 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
   private applySessionConfig(): void {
     const reconnected = this.opened;
     this.opened = true;
+    const session = this.config.buildSessionUpdate();
+    const extra =
+      typeof this.config.sessionExtra === "function"
+        ? this.config.sessionExtra()
+        : this.config.sessionExtra;
     this.socket?.send({
       type: "session.update",
-      session: this.config.buildSessionUpdate(),
+      session: extra ? { ...session, ...extra } : session,
     });
     // G4 resume-by-replay: re-create the prior conversation as items. Providers
     // without native resume start every (re)connection amnesiac; replaying the
@@ -171,6 +181,17 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
     this.requestResponseCreate();
   }
 
+  injectContext(text: string): void {
+    this.requireSocket().send({
+      type: "conversation.item.create",
+      item: {
+        type: "message",
+        role: "system",
+        content: [{ type: "input_text", text }],
+      },
+    });
+  }
+
   requestResponse(): void {
     this.requireSocket().send({ type: "input_audio_buffer.commit" });
     this.requestResponseCreate();
@@ -311,6 +332,9 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
       case "input_audio_buffer.speech_started":
         this.stream.push({ type: "speech_started" });
         break;
+      case "input_audio_buffer.speech_stopped":
+        this.stream.push({ type: "speech_stopped" });
+        break;
       case "response.output_audio_transcript.delta": {
         const delta = msg["delta"];
         if (typeof delta === "string" && delta.length > 0) {
@@ -374,7 +398,10 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
         if (toolCall) {
           this.stream.push(toolCall);
         }
-        this.stream.push({ type: "response_done" });
+        // response.usage is snake_case token counts; forward so the bridge can meter
+        // the native front (previously native turns produced NO usage at all).
+        const usage = extractRealtimeUsage(msg["response"]);
+        this.stream.push(usage ? { type: "response_done", usage } : { type: "response_done" });
         break;
       }
       case "error": {
@@ -385,6 +412,14 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
             : this.config.defaultErrorMessage;
         const code =
           errObj && typeof errObj === "object" ? (errObj as Record<string, unknown>)["code"] : undefined;
+        // A `response.cancel` that raced a just-completed response ("no active response found")
+        // is benign — the cancel is a server-side no-op. This is the multi-turn barge-in cancel
+        // race (a new turn's speech_started fires a cancel as the prior response completes). Do
+        // NOT surface it: it would trip strict consumers and pollute error-rate monitoring.
+        const codeStr = typeof code === "string" ? code : "";
+        if (codeStr === "response_cancel_not_active" || /no active response|cancellation failed/i.test(message)) {
+          break;
+        }
         const recoverable = code !== "invalid_api_key" && code !== "authentication_failed";
         this.stream.push({
           type: "error",
@@ -399,6 +434,21 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
   }
 }
 
+/** Pull snake_case token counts off a realtime response.usage object, when present. */
+function extractRealtimeUsage(response: unknown): RealtimeUsage | undefined {
+  if (!response || typeof response !== "object") return undefined;
+  const usage = (response as Record<string, unknown>)["usage"];
+  if (!usage || typeof usage !== "object") return undefined;
+  const u = usage as Record<string, unknown>;
+  const num = (v: unknown): number | undefined => (typeof v === "number" ? v : undefined);
+  const out: RealtimeUsage = {
+    ...(num(u["input_tokens"]) !== undefined ? { inputTokens: num(u["input_tokens"]) } : {}),
+    ...(num(u["output_tokens"]) !== undefined ? { outputTokens: num(u["output_tokens"]) } : {}),
+    ...(num(u["total_tokens"]) !== undefined ? { totalTokens: num(u["total_tokens"]) } : {}),
+  };
+  return Object.keys(out).length > 0 ? out : undefined;
+}
+
 function extractFunctionCall(response: unknown): RealtimeEvent | null {
   if (!response || typeof response !== "object") return null;
   const output = (response as Record<string, unknown>)["output"];

package/src/realtime-adapter.ts

@@ -33,6 +33,8 @@ export interface RealtimeAdapter {
    * provider cannot accept text input omit it, and the bridge silently ignores typed turns for them.
    */
   sendText?(text: string): void;
+  /** Inject transient context without requesting a provider response. */
+  injectContext?(text: string): void;
   /**
    * Commit any buffered user input and request a response. For Syrinx-OWNED turn detection
    * (provider server VAD disabled via turnDetection:null): the host calls this when its own
@@ -64,13 +66,21 @@ export interface RealtimeResumeMessage {
   readonly content: string;
 }
 
+/** Token usage a realtime provider reports on response completion (e.g. OpenAI response.done). */
+export interface RealtimeUsage {
+  readonly inputTokens?: number;
+  readonly outputTokens?: number;
+  readonly totalTokens?: number;
+}
+
 export type RealtimeEvent =
   | { type: "audio"; pcm16: Uint8Array; sampleRateHz: number }
   | { type: "speech_started" }
+  | { type: "speech_stopped" }
   | { type: "transcript"; role: "user" | "assistant"; text: string; final: boolean }
   | { type: "tool_call"; toolId: string; toolName: string; args: Record<string, unknown> }
   | { type: "response_started" }
-  | { type: "response_done" }
+  | { type: "response_done"; usage?: RealtimeUsage }
   // G4: a native-resume provider issued a fresh resumption handle — persist the
   // latest one and pass it back on reconnect (Gemini `sessionResumption`).
   | { type: "resumption_handle"; handle: string }