@absolutejs/voice 0.0.22-beta.127 → 0.0.22-beta.128

package/README.md

@@ -1324,6 +1324,59 @@ app.use(
 
 Client state now exposes `assistantAudio` on the stream/controller helpers, so apps can buffer or play synthesized chunks without inventing a second transport.
 
+## OpenAI Realtime
+
+Use `createOpenAIRealtimeAdapter(...)` when you want a direct OpenAI Realtime speech-to-speech output path for live smoke tests, duplex benchmarks, or custom realtime orchestration. It implements the same `RealtimeAdapter` contract used by the benchmark harness, so the provider can stream `response.output_audio.delta` audio chunks into AbsoluteJS voice events while still emitting normalized transcript, error, and close events.
+
+```ts
+import { createOpenAIRealtimeAdapter } from '@absolutejs/voice';
+import { runTTSAdapterFixture } from '@absolutejs/voice/testing';
+
+const realtime = createOpenAIRealtimeAdapter({
+	apiKey: process.env.OPENAI_API_KEY!,
+	instructions: 'Answer in one concise sentence.',
+	model: 'gpt-realtime',
+	voice: 'marin'
+});
+
+app.use(
+	voice({
+		path: '/voice',
+		realtime,
+		realtimeInputFormat: {
+			channels: 1,
+			container: 'raw',
+			encoding: 'pcm_s16le',
+			sampleRateHz: 24000
+		},
+		session,
+		onTurn: async ({ turn }) => ({
+			assistantText: `You said: ${turn.text}`
+		}),
+		onComplete: async () => {}
+	})
+);
+
+const report = await runTTSAdapterFixture(
+	realtime,
+	{
+		id: 'openai-realtime-smoke',
+		text: 'Say exactly: AbsoluteJS realtime is online.',
+		title: 'OpenAI Realtime smoke'
+	},
+	{
+		realtimeFormat: {
+			channels: 1,
+			container: 'raw',
+			encoding: 'pcm_s16le',
+			sampleRateHz: 24000
+		}
+	}
+);
+```
+
+For server-to-server use, the adapter opens a WebSocket to OpenAI, sends `session.update`, streams text or base64 PCM input, and emits raw 24kHz mono `pcm_s16le` assistant audio. It requires raw 24kHz mono PCM input because that is the OpenAI Realtime PCM format. The main `voice(...)` route can now run in cascaded mode with `stt` plus optional `tts`, or direct realtime mode with `realtime`. Browser demos should make sure the captured PCM format matches `realtimeInputFormat` or resample before sending audio.
+
 If you want a minimal browser playback path, use the client audio player:
 
 ```ts

package/dist/index.d.ts

@@ -31,6 +31,7 @@ export { createVoicePhoneAgent } from './phoneAgent';
 export { createStoredVoiceCallReviewArtifact, createStoredVoiceExternalObjectMap, createStoredVoiceIntegrationEvent, createStoredVoiceOpsTask, createVoiceFileExternalObjectMapStore, createVoiceFileAssistantMemoryStore, createVoiceFileAuditEventStore, createVoiceFileAuditSinkDeliveryStore, createVoiceFileCampaignStore, createVoiceFileIntegrationEventStore, createVoiceFileReviewStore, createVoiceFileRuntimeStorage, createVoiceFileSessionStore, createVoiceFileTaskStore, createVoiceFileTraceSinkDeliveryStore, createVoiceFileTraceEventStore } from './fileStore';
 export { createVoiceAssistantMemoryHandle, createVoiceAssistantMemoryRecord, createVoiceMemoryAssistantMemoryStore, resolveVoiceAssistantMemoryNamespace } from './assistantMemory';
 export { createAnthropicVoiceAssistantModel, createGeminiVoiceAssistantModel, createJSONVoiceAssistantModel, createOpenAIVoiceAssistantModel, resolveVoiceProviderRoutingPolicyPreset, createVoiceProviderRouter } from './modelAdapters';
+export { createOpenAIRealtimeAdapter } from './openaiRealtime';
 export { createOpenAIVoiceTTS } from './openaiTTS';
 export { createVoiceProviderHealthHTMLHandler, createVoiceProviderHealthJSONHandler, createVoiceProviderHealthRoutes, renderVoiceProviderHealthHTML, summarizeVoiceProviderHealth } from './providerHealth';
 export { createVoiceProviderCapabilityHTMLHandler, createVoiceProviderCapabilityJSONHandler, createVoiceProviderCapabilityRoutes, renderVoiceProviderCapabilityHTML, summarizeVoiceProviderCapabilities } from './providerCapabilities';
@@ -81,6 +82,7 @@ export type { VoiceWorkflowContract, VoiceWorkflowContractDefinition, VoiceWorkf
 export type { VoiceSessionListHTMLHandlerOptions, VoiceSessionListItem, VoiceSessionListOptions, VoiceSessionListRoutesOptions, VoiceSessionListStatus, VoiceSessionReplay, VoiceSessionReplayHTMLHandlerOptions, VoiceSessionReplayOptions, VoiceSessionReplayRoutesOptions, VoiceSessionReplayTurn } from './sessionReplay';
 export type { AnthropicVoiceAssistantModelOptions, GeminiVoiceAssistantModelOptions, OpenAIVoiceAssistantModelOptions, VoiceProviderRouterEvent, VoiceProviderRouterFallbackMode, VoiceProviderRouterHealthOptions, VoiceProviderRouterOptions, VoiceProviderRouterPolicy, VoiceProviderRouterPolicyPreset, VoiceProviderRouterPolicyWeights, VoiceProviderRouterProviderHealth, VoiceProviderRouterProviderProfile, VoiceProviderRouterStrategy, VoiceJSONAssistantModelHandler, VoiceJSONAssistantModelOptions } from './modelAdapters';
 export type { OpenAIVoiceTTSOptions, OpenAIVoiceTTSVoice } from './openaiTTS';
+export type { OpenAIRealtimeAdapterOptions, OpenAIRealtimeModel, OpenAIRealtimeNoiseReduction, OpenAIRealtimeResponseMode, OpenAIRealtimeTranscriptionModel, OpenAIRealtimeVoice } from './openaiRealtime';
 export type { VoiceProviderHealthStatus, VoiceProviderHealthSummary, VoiceProviderHealthSummaryOptions } from './providerHealth';
 export type { VoiceProviderCapabilityDefinition, VoiceProviderCapabilityHandlerOptions, VoiceProviderCapabilityHTMLHandlerOptions, VoiceProviderCapabilityKind, VoiceProviderCapabilityOptions, VoiceProviderCapabilityReport, VoiceProviderCapabilityRoutesOptions, VoiceProviderCapabilitySummary } from './providerCapabilities';
 export type { VoiceProviderRoutingContractDefinition, VoiceProviderRoutingContractIssue, VoiceProviderRoutingContractReport, VoiceProviderRoutingContractRunOptions, VoiceProviderRoutingExpectation, VoiceProviderRoutingStatus } from './providerRoutingContract';

package/dist/index.js

@@ -3413,6 +3413,12 @@ var DEFAULT_FORMAT = {
   encoding: "pcm_s16le",
   sampleRateHz: 16000
 };
+var DEFAULT_REALTIME_FORMAT = {
+  channels: 1,
+  container: "raw",
+  encoding: "pcm_s16le",
+  sampleRateHz: 24000
+};
 var toError = (value) => value instanceof Error ? value : new Error(String(value));
 var createEmptyCurrentTurn = () => ({
   finalText: "",
@@ -3793,6 +3799,23 @@ var createVoiceSession = (options) => {
       });
     }
   };
+  const sendAssistantAudio = async (chunk, input) => {
+    const normalizedChunk = chunk instanceof Uint8Array ? new Uint8Array(chunk) : chunk instanceof ArrayBuffer ? new Uint8Array(chunk.slice(0)) : new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
+    await send({
+      chunkBase64: encodeBase64(normalizedChunk),
+      format: input.format,
+      receivedAt: input.receivedAt,
+      turnId: activeTTSTurnId,
+      type: "audio"
+    });
+    if (activeTTSTurnId) {
+      await appendTurnLatencyStage({
+        at: input.receivedAt,
+        stage: "assistant_audio_received",
+        turnId: activeTTSTurnId
+      });
+    }
+  };
   const scheduleTurnCommit = (delayMs, reason, reset = true) => {
     if (!reset && silenceTimer) {
       return;
@@ -4494,8 +4517,12 @@ var createVoiceSession = (options) => {
     if (sttSession) {
       return sttSession;
     }
-    const openedSession = await options.stt.open({
-      format: DEFAULT_FORMAT,
+    const inputAdapter = options.realtime ?? options.stt;
+    if (!inputAdapter) {
+      throw new Error("Voice session requires either an stt or realtime adapter.");
+    }
+    const openedSession = await inputAdapter.open({
+      format: options.realtime ? options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT : DEFAULT_FORMAT,
       languageStrategy: options.languageStrategy,
       lexicon,
       phraseHints,
@@ -4530,6 +4557,16 @@ var createVoiceSession = (options) => {
     openedSession.on("close", (event) => {
       runAdapterEvent("adapter.close", () => handleClose(event));
     });
+    if (options.realtime) {
+      openedSession.on("audio", ({ chunk, format, receivedAt }) => {
+        runAdapterEvent("adapter.audio", async () => {
+          await sendAssistantAudio(chunk, {
+            format,
+            receivedAt
+          });
+        });
+      });
+    }
     return openedSession;
   };
   const ensureTTSSession = async () => {
@@ -4554,21 +4591,10 @@ var createVoiceSession = (options) => {
           if (ttsSession !== openedSession) {
             return;
           }
-          const normalizedChunk = chunk instanceof Uint8Array ? new Uint8Array(chunk) : chunk instanceof ArrayBuffer ? new Uint8Array(chunk.slice(0)) : new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
-          await send({
-            chunkBase64: encodeBase64(normalizedChunk),
+          await sendAssistantAudio(chunk, {
             format,
-            receivedAt,
-            turnId: activeTTSTurnId,
-            type: "audio"
+            receivedAt
           });
-          if (activeTTSTurnId) {
-            await appendTurnLatencyStage({
-              at: receivedAt,
-              stage: "assistant_audio_received",
-              turnId: activeTTSTurnId
-            });
-          }
         });
       });
       openedSession.on("error", (event) => {
@@ -4647,7 +4673,8 @@ var createVoiceSession = (options) => {
       await appendTrace({
         payload: {
           text: output.assistantText,
-          ttsConfigured: Boolean(options.tts)
+          ttsConfigured: Boolean(options.tts),
+          realtimeConfigured: Boolean(options.realtime)
         },
         session,
         turnId: turn.id,
@@ -4679,9 +4706,35 @@ var createVoiceSession = (options) => {
             turnId: turn.id,
             type: "turn.assistant"
           });
+        } else if (options.realtime) {
+          const activeRealtimeSession = await ensureAdapter();
+          const realtimeStartedAt = Date.now();
+          activeTTSTurnId = turn.id;
+          await appendTurnLatencyStage({
+            at: realtimeStartedAt,
+            session,
+            stage: "tts_send_started",
+            turnId: turn.id
+          });
+          await activeRealtimeSession.send(output.assistantText);
+          await appendTurnLatencyStage({
+            session,
+            stage: "tts_send_completed",
+            turnId: turn.id
+          });
+          await appendTrace({
+            payload: {
+              elapsedMs: Date.now() - realtimeStartedAt,
+              mode: "realtime",
+              status: "sent"
+            },
+            session,
+            turnId: turn.id,
+            type: "turn.assistant"
+          });
         }
       } catch (error) {
-        logger.warn("voice tts send failed", {
+        logger.warn("voice assistant audio send failed", {
           error: toError(error).message,
           sessionId: options.id,
           turnId: turn.id
@@ -4689,7 +4742,7 @@ var createVoiceSession = (options) => {
         await appendTrace({
           payload: {
             error: toError(error).message,
-            status: "tts-send-failed"
+            status: options.realtime ? "realtime-send-failed" : "tts-send-failed"
           },
           session,
           turnId: turn.id,
@@ -4894,7 +4947,7 @@ var createVoiceSession = (options) => {
       turn,
       type: "turn"
     });
-    if (options.sttLifecycle === "turn-scoped") {
+    if (options.stt && options.sttLifecycle === "turn-scoped") {
       await closeAdapter("turn-commit");
     }
     await completeTurn(updatedSession, turn);
@@ -5307,6 +5360,9 @@ var resolveLexicon = async (config, input) => {
   return normalizeLexicon(config.lexicon);
 };
 var voice = (config) => {
+  if (!config.stt && !config.realtime) {
+    throw new Error("voice requires either an stt or realtime adapter.");
+  }
   const runtime = {
     activeSessions: new Map,
     logger: resolveLogger(config.logger),
@@ -5381,6 +5437,8 @@ var voice = (config) => {
       socket: createSocketAdapter(ws),
       store: config.session,
       trace: config.trace,
+      realtime: config.realtime,
+      realtimeInputFormat: config.realtimeInputFormat,
       stt: config.stt,
       sttFallback: sessionOptions.sttFallback,
       sttLifecycle: sessionOptions.sttLifecycle,
@@ -17088,13 +17146,517 @@ var createGeminiVoiceAssistantModel = (options) => {
     }
   };
 };
-// src/openaiTTS.ts
+// src/openaiRealtime.ts
+var DEFAULT_AUTO_COMMIT_SILENCE_MS = 450;
+var DEFAULT_BASE_URL = "wss://api.openai.com/v1/realtime";
+var DEFAULT_MODEL = "gpt-realtime";
+var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-mini-transcribe";
+var DEFAULT_VOICE = "marin";
 var OPENAI_PCM24_FORMAT = {
   channels: 1,
   container: "raw",
   encoding: "pcm_s16le",
   sampleRateHz: 24000
 };
+var createListenerMap = () => ({
+  audio: new Set,
+  close: new Set,
+  endOfTurn: new Set,
+  error: new Set,
+  final: new Set,
+  partial: new Set
+});
+var emit = async (listeners, event, payload) => {
+  for (const listener of listeners[event]) {
+    await listener(payload);
+  }
+};
+var compact = (value) => Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
+var resolveErrorMessage = (error) => {
+  if (typeof error === "string" && error.trim()) {
+    return error;
+  }
+  if (error instanceof Error && error.message.trim()) {
+    return error.message;
+  }
+  if (error && typeof error === "object") {
+    const record = error;
+    for (const key of ["message", "reason", "description", "detail"]) {
+      const candidate = record[key];
+      if (typeof candidate === "string" && candidate.trim()) {
+        return candidate;
+      }
+    }
+    if ("error" in record) {
+      return resolveErrorMessage(record.error);
+    }
+    try {
+      return JSON.stringify(error);
+    } catch {}
+  }
+  return "OpenAI realtime error";
+};
+var toUint8Array2 = (value) => value instanceof ArrayBuffer ? new Uint8Array(value) : new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+var toBase643 = (value) => Buffer.from(toUint8Array2(value)).toString("base64");
+var textTranscript = (text) => ({
+  id: `openai-realtime-text-${crypto.randomUUID()}`,
+  isFinal: true,
+  text,
+  vendor: "openai"
+});
+var audioTranscript = (itemId, text, isFinal) => ({
+  id: itemId,
+  isFinal,
+  text,
+  vendor: "openai"
+});
+var assertPCM24Mono = (format) => {
+  if (format.container !== "raw" || format.encoding !== "pcm_s16le" || format.sampleRateHz !== 24000 || format.channels !== 1) {
+    throw new Error("OpenAI Realtime requires raw pcm_s16le audio at 24kHz mono.");
+  }
+};
+var resolveTranscriptionLanguage = (options, openOptions) => {
+  if (options.inputTranscriptionLanguage?.trim()) {
+    return options.inputTranscriptionLanguage.trim();
+  }
+  if (openOptions.languageStrategy?.mode !== "fixed") {
+    return;
+  }
+  const language = openOptions.languageStrategy.primaryLanguage.trim();
+  return language.length > 0 ? language : undefined;
+};
+var phraseHintPrompt = (options) => {
+  const terms = (options.phraseHints ?? []).flatMap((hint) => [
+    hint.text,
+    ...hint.aliases ?? []
+  ]);
+  const unique = terms.filter((value, index) => terms.indexOf(value) === index);
+  return unique.length ? `Prioritize accurate recovery of these phrases when heard: ${unique.join(", ")}.` : undefined;
+};
+var lexiconPrompt = (options) => {
+  const entries = (options.lexicon ?? []).flatMap((entry) => {
+    const details = [
+      entry.text,
+      entry.pronunciation ? `pronounced ${entry.pronunciation}` : undefined,
+      entry.aliases?.length ? `may also sound like ${entry.aliases.join(", ")}` : undefined,
+      entry.language ? `language ${entry.language}` : undefined
+    ].filter((value) => !!value);
+    return details.length ? [details.join(" - ")] : [];
+  });
+  return entries.length ? `Use this pronunciation lexicon when transcribing: ${entries.join("; ")}.` : undefined;
+};
+var withOpenPrompts = (options, openOptions) => {
+  const phraseHints = phraseHintPrompt(openOptions);
+  const lexicon = lexiconPrompt(openOptions);
+  if (!phraseHints && !lexicon) {
+    return options;
+  }
+  return {
+    ...options,
+    inputTranscriptionPrompt: [
+      options.inputTranscriptionPrompt,
+      phraseHints,
+      lexicon
+    ].filter((value) => !!value?.trim()).join(`
+
+`)
+  };
+};
+var sessionUpdateEvent = (options, openOptions) => {
+  const responseMode = options.responseMode ?? "audio";
+  const language = resolveTranscriptionLanguage(options, openOptions);
+  const transcription = options.inputTranscriptionModel === null ? null : compact({
+    language,
+    model: options.inputTranscriptionModel ?? DEFAULT_TRANSCRIPTION_MODEL,
+    prompt: options.inputTranscriptionPrompt
+  });
+  return {
+    event_id: `session-update-${crypto.randomUUID()}`,
+    session: compact({
+      audio: {
+        input: compact({
+          format: {
+            rate: 24000,
+            type: "audio/pcm"
+          },
+          noise_reduction: options.noiseReduction ? { type: options.noiseReduction } : undefined,
+          transcription,
+          turn_detection: null
+        }),
+        output: responseMode === "audio" ? compact({
+          format: {
+            rate: 24000,
+            type: "audio/pcm"
+          },
+          speed: options.speed,
+          voice: options.voice ?? DEFAULT_VOICE
+        }) : undefined
+      },
+      instructions: options.instructions,
+      max_output_tokens: options.maxOutputTokens,
+      output_modalities: [responseMode],
+      temperature: options.temperature,
+      type: "realtime"
+    }),
+    type: "session.update"
+  };
+};
+var responseCreateEvent = (options) => {
+  const responseMode = options.responseMode ?? "audio";
+  return {
+    response: compact({
+      audio: responseMode === "audio" ? {
+        output: compact({
+          format: {
+            rate: 24000,
+            type: "audio/pcm"
+          },
+          voice: options.voice ?? DEFAULT_VOICE
+        })
+      } : undefined,
+      conversation: "auto",
+      max_output_tokens: options.maxOutputTokens,
+      output_modalities: [responseMode]
+    }),
+    type: "response.create"
+  };
+};
+var createOpenAIRealtimeAdapter = (options) => {
+  const baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
+  const Socket = options.webSocket ?? globalThis.WebSocket;
+  return {
+    kind: "realtime",
+    open: (openOptions) => {
+      assertPCM24Mono(openOptions.format);
+      const runtimeOptions = openOptions;
+      const runtimeConfig = withOpenPrompts(options, runtimeOptions);
+      const model = runtimeConfig.model ?? DEFAULT_MODEL;
+      const listeners = createListenerMap();
+      const socket = new Socket(`${baseUrl.replace(/\/$/, "")}?model=${encodeURIComponent(model)}`, {
+        headers: {
+          Authorization: `Bearer ${runtimeConfig.apiKey}`
+        }
+      });
+      const primaryUpdate = sessionUpdateEvent(runtimeConfig, runtimeOptions);
+      const pendingMessages = [];
+      const partials = new Map;
+      const finals = new Set;
+      const autoCommitSilenceMs = runtimeConfig.autoCommitSilenceMs ?? DEFAULT_AUTO_COMMIT_SILENCE_MS;
+      let audioCommitTimer;
+      let closeEmitted = false;
+      let closed = false;
+      let pendingAudio = false;
+      let ready = false;
+      let readyTimeout;
+      let socketOpen = false;
+      let resolveReady;
+      let rejectReady;
+      const readyPromise = new Promise((resolve2, reject) => {
+        resolveReady = resolve2;
+        rejectReady = reject;
+      });
+      const clearReadyTimeout = () => {
+        if (readyTimeout) {
+          clearTimeout(readyTimeout);
+          readyTimeout = undefined;
+        }
+      };
+      const markReady = () => {
+        if (ready || closed) {
+          return;
+        }
+        ready = true;
+        clearReadyTimeout();
+        resolveReady();
+      };
+      const failReady = (error) => {
+        if (ready || closed) {
+          return;
+        }
+        clearReadyTimeout();
+        rejectReady(error);
+      };
+      const sendRaw = (payload) => {
+        const serialized = JSON.stringify(payload);
+        if (!socketOpen) {
+          pendingMessages.push(serialized);
+          return;
+        }
+        socket.send(serialized);
+      };
+      const flush = () => {
+        for (const message of pendingMessages.splice(0)) {
+          socket.send(message);
+        }
+      };
+      const emitClose = async (code, reason, recoverable = false) => {
+        if (closeEmitted) {
+          return;
+        }
+        closeEmitted = true;
+        await emit(listeners, "close", {
+          code,
+          reason,
+          recoverable,
+          type: "close"
+        });
+      };
+      const commitAudio = async () => {
+        if (closed || !pendingAudio) {
+          return;
+        }
+        pendingAudio = false;
+        sendRaw({ type: "input_audio_buffer.commit" });
+        sendRaw(responseCreateEvent(runtimeConfig));
+      };
+      const resetAudioTimer = () => {
+        if (audioCommitTimer) {
+          clearTimeout(audioCommitTimer);
+        }
+        audioCommitTimer = setTimeout(() => {
+          commitAudio();
+        }, autoCommitSilenceMs);
+      };
+      socket.addEventListener("open", () => {
+        socketOpen = true;
+        sendRaw(primaryUpdate);
+        flush();
+        readyTimeout = setTimeout(() => {
+          failReady(new Error("OpenAI realtime session did not become ready."));
+        }, 8000);
+      }, { once: true });
+      socket.addEventListener("message", (event) => {
+        try {
+          const payload = JSON.parse(String(event.data));
+          const shouldEmitResponseTranscripts = runtimeConfig.emitResponseTranscripts === true;
+          switch (payload.type) {
+            case "session.created":
+            case "session.updated":
+              markReady();
+              return;
+            case "conversation.item.input_audio_transcription.delta": {
+              const itemId = typeof payload.item_id === "string" ? payload.item_id : undefined;
+              const delta = typeof payload.delta === "string" ? payload.delta : undefined;
+              if (!itemId || !delta) {
+                return;
+              }
+              const text = `${partials.get(itemId) ?? ""}${delta}`;
+              partials.set(itemId, text);
+              emit(listeners, "partial", {
+                receivedAt: Date.now(),
+                transcript: audioTranscript(itemId, text, false),
+                type: "partial"
+              });
+              return;
+            }
+            case "conversation.item.input_audio_transcription.completed": {
+              const itemId = typeof payload.item_id === "string" ? payload.item_id : undefined;
+              const transcript = typeof payload.transcript === "string" ? payload.transcript : undefined;
+              if (!itemId || !transcript || finals.has(itemId)) {
+                return;
+              }
+              finals.add(itemId);
+              partials.set(itemId, transcript);
+              emit(listeners, "final", {
+                receivedAt: Date.now(),
+                transcript: audioTranscript(itemId, transcript, true),
+                type: "final"
+              });
+              emit(listeners, "endOfTurn", {
+                receivedAt: Date.now(),
+                reason: "vendor",
+                type: "endOfTurn"
+              });
+              return;
+            }
+            case "conversation.item.input_audio_transcription.failed": {
+              const error = payload.error && typeof payload.error === "object" ? payload.error : undefined;
+              emit(listeners, "error", {
+                code: error?.code,
+                error: new Error(resolveErrorMessage(error ?? payload)),
+                recoverable: true,
+                type: "error"
+              });
+              return;
+            }
+            case "response.audio.delta":
+            case "response.output_audio.delta": {
+              const delta = typeof payload.delta === "string" ? payload.delta : undefined;
+              if (!delta) {
+                return;
+              }
+              emit(listeners, "audio", {
+                chunk: Buffer.from(delta, "base64"),
+                format: OPENAI_PCM24_FORMAT,
+                receivedAt: Date.now(),
+                type: "audio"
+              });
+              return;
+            }
+            case "response.audio_transcript.delta":
+            case "response.output_audio_transcript.delta":
+            case "response.output_text.delta": {
+              if (!shouldEmitResponseTranscripts) {
+                return;
+              }
+              const delta = typeof payload.delta === "string" ? payload.delta : undefined;
+              if (!delta) {
+                return;
+              }
+              emit(listeners, "partial", {
+                receivedAt: Date.now(),
+                transcript: textTranscript(delta),
+                type: "partial"
+              });
+              return;
+            }
+            case "response.audio_transcript.done":
+            case "response.output_audio_transcript.done":
+            case "response.output_text.done": {
+              if (!shouldEmitResponseTranscripts) {
+                return;
+              }
+              const transcript = typeof payload.transcript === "string" ? payload.transcript : undefined;
+              if (!transcript) {
+                return;
+              }
+              emit(listeners, "final", {
+                receivedAt: Date.now(),
+                transcript: textTranscript(transcript),
+                type: "final"
+              });
+              emit(listeners, "endOfTurn", {
+                receivedAt: Date.now(),
+                reason: "vendor",
+                type: "endOfTurn"
+              });
+              return;
+            }
+            case "error": {
+              const error = payload.error && typeof payload.error === "object" ? payload.error : {};
+              const message = resolveErrorMessage(error);
+              emit(listeners, "error", {
+                code: error.code,
+                error: new Error(message),
+                recoverable: true,
+                type: "error"
+              });
+              if (!ready && error.event_id === primaryUpdate.event_id) {
+                failReady(new Error(message));
+              }
+              return;
+            }
+            default:
+              return;
+          }
+        } catch (error) {
+          emit(listeners, "error", {
+            error: new Error(resolveErrorMessage(error)),
+            recoverable: true,
+            type: "error"
+          });
+        }
+      });
+      socket.addEventListener("error", (event) => {
+        const error = new Error(resolveErrorMessage(event));
+        failReady(error);
+        emit(listeners, "error", {
+          error,
+          recoverable: false,
+          type: "error"
+        });
+      });
+      socket.addEventListener("close", (event) => {
+        socketOpen = false;
+        clearReadyTimeout();
+        if (!ready) {
+          failReady(new Error("OpenAI realtime session closed before ready."));
+        }
+        emitClose(event.code, event.reason || undefined, event.code !== 1000);
+      });
+      if (openOptions.signal) {
+        if (openOptions.signal.aborted) {
+          closed = true;
+          socket.close(1000, "aborted");
+        } else {
+          openOptions.signal.addEventListener("abort", () => {
+            if (!closed) {
+              closed = true;
+              socket.close(1000, "aborted");
+            }
+          }, { once: true });
+        }
+      }
+      return {
+        close: async (reason) => {
+          if (closed) {
+            return;
+          }
+          closed = true;
+          clearReadyTimeout();
+          if (audioCommitTimer) {
+            clearTimeout(audioCommitTimer);
+            audioCommitTimer = undefined;
+          }
+          await commitAudio().catch(() => {});
+          socket.close(1000, reason);
+          await emitClose(1000, reason, false);
+        },
+        on: (event, handler) => {
+          listeners[event].add(handler);
+          return () => {
+            listeners[event].delete(handler);
+          };
+        },
+        send: async (input) => {
+          await readyPromise;
+          if (closed) {
+            return;
+          }
+          if (typeof input === "string") {
+            const text = input.trim();
+            if (!text) {
+              return;
+            }
+            await emit(listeners, "final", {
+              receivedAt: Date.now(),
+              transcript: textTranscript(text),
+              type: "final"
+            });
+            await emit(listeners, "endOfTurn", {
+              receivedAt: Date.now(),
+              reason: "manual",
+              type: "endOfTurn"
+            });
+            sendRaw({
+              item: {
+                content: [{ text, type: "input_text" }],
+                role: "user",
+                type: "message"
+              },
+              type: "conversation.item.create"
+            });
+            sendRaw(responseCreateEvent(runtimeConfig));
+            return;
+          }
+          sendRaw({
+            audio: toBase643(input),
+            type: "input_audio_buffer.append"
+          });
+          pendingAudio = true;
+          resetAudioTimer();
+        }
+      };
+    }
+  };
+};
+// src/openaiTTS.ts
+var OPENAI_PCM24_FORMAT2 = {
+  channels: 1,
+  container: "raw",
+  encoding: "pcm_s16le",
+  sampleRateHz: 24000
+};
 var resolveInstructions = async (instructions, input) => {
   if (typeof instructions === "function") {
     return instructions(input);
@@ -17102,7 +17664,7 @@ var resolveInstructions = async (instructions, input) => {
   return instructions;
 };
 var createTTSHTTPError = (response) => new Error(`OpenAI voice TTS failed: HTTP ${response.status}`);
-var emit = async (listeners, event, payload) => {
+var emit2 = async (listeners, event, payload) => {
   for (const handler of listeners[event]) {
     await Promise.resolve(handler(payload));
   }
@@ -17132,7 +17694,7 @@ var createOpenAIVoiceTTS = (options) => {
           closed = true;
           abortController.abort();
           openOptions.signal?.removeEventListener("abort", signalAbort);
-          await emit(listeners, "close", {
+          await emit2(listeners, "close", {
             reason,
             type: "close"
           });
@@ -17175,9 +17737,9 @@ var createOpenAIVoiceTTS = (options) => {
             if (!response.body) {
               const chunk = new Uint8Array(await response.arrayBuffer());
               if (!closed && chunk.byteLength > 0) {
-                await emit(listeners, "audio", {
+                await emit2(listeners, "audio", {
                   chunk,
-                  format: OPENAI_PCM24_FORMAT,
+                  format: OPENAI_PCM24_FORMAT2,
                   receivedAt: Date.now(),
                   type: "audio"
                 });
@@ -17192,9 +17754,9 @@ var createOpenAIVoiceTTS = (options) => {
                   break;
                 }
                 if (value.byteLength > 0) {
-                  await emit(listeners, "audio", {
+                  await emit2(listeners, "audio", {
                     chunk: new Uint8Array(value),
-                    format: OPENAI_PCM24_FORMAT,
+                    format: OPENAI_PCM24_FORMAT2,
                     receivedAt: Date.now(),
                     type: "audio"
                   });
@@ -17208,7 +17770,7 @@ var createOpenAIVoiceTTS = (options) => {
               return;
             }
             const normalizedError = error instanceof Error ? error : new Error(String(error));
-            await emit(listeners, "error", {
+            await emit2(listeners, "error", {
               error: normalizedError,
               recoverable: true,
               type: "error"
@@ -19778,11 +20340,11 @@ var createResolver = (options) => {
       selectedProvider: preferred
     };
   };
-  const emit2 = async (event, input) => {
+  const emit3 = async (event, input) => {
     await options.onProviderEvent?.(event, input);
   };
   return {
-    emit: emit2,
+    emit: emit3,
     getSuppressionRemainingMs,
     providerIds,
     recordError,
@@ -22301,6 +22863,7 @@ export {
   createPhraseHintCorrectionHandler,
   createOpenAIVoiceTTS,
   createOpenAIVoiceAssistantModel,
+  createOpenAIRealtimeAdapter,
   createMemoryVoiceTelephonyWebhookIdempotencyStore,
   createJSONVoiceAssistantModel,
   createId,

package/dist/openaiRealtime.d.ts

@@ -0,0 +1,27 @@
+import type { RealtimeAdapter } from './types';
+export type OpenAIRealtimeModel = 'gpt-realtime' | 'gpt-realtime-mini' | 'gpt-4o-realtime-preview' | 'gpt-4o-mini-realtime-preview' | (string & {});
+export type OpenAIRealtimeVoice = 'alloy' | 'ash' | 'ballad' | 'cedar' | 'coral' | 'echo' | 'marin' | 'sage' | 'shimmer' | 'verse' | {
+    id: string;
+} | (string & {});
+export type OpenAIRealtimeTranscriptionModel = 'gpt-4o-mini-transcribe' | 'gpt-4o-transcribe' | 'whisper-1' | (string & {});
+export type OpenAIRealtimeNoiseReduction = 'near_field' | 'far_field';
+export type OpenAIRealtimeResponseMode = 'audio' | 'text';
+export type OpenAIRealtimeAdapterOptions = {
+    apiKey: string;
+    autoCommitSilenceMs?: number;
+    baseUrl?: string;
+    emitResponseTranscripts?: boolean;
+    inputTranscriptionLanguage?: string;
+    inputTranscriptionModel?: OpenAIRealtimeTranscriptionModel | null;
+    inputTranscriptionPrompt?: string;
+    instructions?: string;
+    maxOutputTokens?: number | 'inf';
+    model?: OpenAIRealtimeModel;
+    noiseReduction?: OpenAIRealtimeNoiseReduction;
+    responseMode?: OpenAIRealtimeResponseMode;
+    speed?: number;
+    temperature?: number;
+    voice?: OpenAIRealtimeVoice;
+    webSocket?: typeof WebSocket;
+};
+export declare const createOpenAIRealtimeAdapter: (options: OpenAIRealtimeAdapterOptions) => RealtimeAdapter;

package/dist/telephony/twilio.d.ts

@@ -2,7 +2,7 @@ import { Elysia } from 'elysia';
 import type { VoiceTelephonySetupStatus, VoiceTelephonySmokeCheck, VoiceTelephonySmokeReport } from './contract';
 import { type VoiceTelephonyOutcomePolicy, type VoiceTelephonyWebhookRoutesOptions } from '../telephonyOutcome';
 import { type VoiceCallReviewArtifact, type VoiceCallReviewConfig } from '../testing/review';
-import type { AudioFormat, VoiceLogger, VoicePluginConfig, VoiceSessionRecord, VoiceServerMessage } from '../types';
+import type { AudioFormat, STTAdapter, VoiceLogger, VoicePluginConfig, VoiceSessionRecord, VoiceServerMessage } from '../types';
 type TwilioMediaPayload = {
     chunk?: string;
     payload: string;
@@ -78,7 +78,7 @@ export type TwilioMediaStreamSocket = {
     close: (code?: number, reason?: string) => void | Promise<void>;
     send: (data: string) => void | Promise<void>;
 };
-export type TwilioMediaStreamBridgeOptions<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = Omit<VoicePluginConfig<TContext, TSession, TResult>, 'htmx' | 'path'> & {
+export type TwilioMediaStreamBridgeOptions<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = Omit<VoicePluginConfig<TContext, TSession, TResult>, 'htmx' | 'path' | 'stt'> & {
     clearOnInboundMedia?: boolean;
     context: TContext;
     logger?: VoiceLogger;
@@ -97,6 +97,7 @@ export type TwilioMediaStreamBridgeOptions<TContext = unknown, TSession extends
     };
     scenarioId?: string;
     sessionId?: string;
+    stt: STTAdapter;
 };
 export type TwilioMediaStreamBridge = {
     close: (reason?: string) => Promise<void>;

package/dist/testing/index.js

@@ -5033,6 +5033,12 @@ var DEFAULT_FORMAT = {
   encoding: "pcm_s16le",
   sampleRateHz: 16000
 };
+var DEFAULT_REALTIME_FORMAT = {
+  channels: 1,
+  container: "raw",
+  encoding: "pcm_s16le",
+  sampleRateHz: 24000
+};
 var toError = (value) => value instanceof Error ? value : new Error(String(value));
 var createEmptyCurrentTurn = () => ({
   finalText: "",
@@ -5413,6 +5419,23 @@ var createVoiceSession = (options) => {
       });
     }
   };
+  const sendAssistantAudio = async (chunk, input) => {
+    const normalizedChunk = chunk instanceof Uint8Array ? new Uint8Array(chunk) : chunk instanceof ArrayBuffer ? new Uint8Array(chunk.slice(0)) : new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
+    await send({
+      chunkBase64: encodeBase64(normalizedChunk),
+      format: input.format,
+      receivedAt: input.receivedAt,
+      turnId: activeTTSTurnId,
+      type: "audio"
+    });
+    if (activeTTSTurnId) {
+      await appendTurnLatencyStage({
+        at: input.receivedAt,
+        stage: "assistant_audio_received",
+        turnId: activeTTSTurnId
+      });
+    }
+  };
   const scheduleTurnCommit = (delayMs, reason, reset = true) => {
     if (!reset && silenceTimer) {
       return;
@@ -6114,8 +6137,12 @@ var createVoiceSession = (options) => {
     if (sttSession) {
       return sttSession;
     }
-    const openedSession = await options.stt.open({
-      format: DEFAULT_FORMAT,
+    const inputAdapter = options.realtime ?? options.stt;
+    if (!inputAdapter) {
+      throw new Error("Voice session requires either an stt or realtime adapter.");
+    }
+    const openedSession = await inputAdapter.open({
+      format: options.realtime ? options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT : DEFAULT_FORMAT,
       languageStrategy: options.languageStrategy,
       lexicon,
       phraseHints,
@@ -6150,6 +6177,16 @@ var createVoiceSession = (options) => {
     openedSession.on("close", (event) => {
       runAdapterEvent("adapter.close", () => handleClose(event));
     });
+    if (options.realtime) {
+      openedSession.on("audio", ({ chunk, format, receivedAt }) => {
+        runAdapterEvent("adapter.audio", async () => {
+          await sendAssistantAudio(chunk, {
+            format,
+            receivedAt
+          });
+        });
+      });
+    }
     return openedSession;
   };
   const ensureTTSSession = async () => {
@@ -6174,21 +6211,10 @@ var createVoiceSession = (options) => {
           if (ttsSession !== openedSession) {
             return;
           }
-          const normalizedChunk = chunk instanceof Uint8Array ? new Uint8Array(chunk) : chunk instanceof ArrayBuffer ? new Uint8Array(chunk.slice(0)) : new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
-          await send({
-            chunkBase64: encodeBase64(normalizedChunk),
+          await sendAssistantAudio(chunk, {
             format,
-            receivedAt,
-            turnId: activeTTSTurnId,
-            type: "audio"
+            receivedAt
           });
-          if (activeTTSTurnId) {
-            await appendTurnLatencyStage({
-              at: receivedAt,
-              stage: "assistant_audio_received",
-              turnId: activeTTSTurnId
-            });
-          }
         });
       });
       openedSession.on("error", (event) => {
@@ -6267,7 +6293,8 @@ var createVoiceSession = (options) => {
       await appendTrace({
         payload: {
           text: output.assistantText,
-          ttsConfigured: Boolean(options.tts)
+          ttsConfigured: Boolean(options.tts),
+          realtimeConfigured: Boolean(options.realtime)
         },
         session,
         turnId: turn.id,
@@ -6299,9 +6326,35 @@ var createVoiceSession = (options) => {
             turnId: turn.id,
             type: "turn.assistant"
           });
+        } else if (options.realtime) {
+          const activeRealtimeSession = await ensureAdapter();
+          const realtimeStartedAt = Date.now();
+          activeTTSTurnId = turn.id;
+          await appendTurnLatencyStage({
+            at: realtimeStartedAt,
+            session,
+            stage: "tts_send_started",
+            turnId: turn.id
+          });
+          await activeRealtimeSession.send(output.assistantText);
+          await appendTurnLatencyStage({
+            session,
+            stage: "tts_send_completed",
+            turnId: turn.id
+          });
+          await appendTrace({
+            payload: {
+              elapsedMs: Date.now() - realtimeStartedAt,
+              mode: "realtime",
+              status: "sent"
+            },
+            session,
+            turnId: turn.id,
+            type: "turn.assistant"
+          });
         }
       } catch (error) {
-        logger.warn("voice tts send failed", {
+        logger.warn("voice assistant audio send failed", {
           error: toError(error).message,
           sessionId: options.id,
           turnId: turn.id
@@ -6309,7 +6362,7 @@ var createVoiceSession = (options) => {
         await appendTrace({
           payload: {
             error: toError(error).message,
-            status: "tts-send-failed"
+            status: options.realtime ? "realtime-send-failed" : "tts-send-failed"
           },
           session,
           turnId: turn.id,
@@ -6514,7 +6567,7 @@ var createVoiceSession = (options) => {
       turn,
       type: "turn"
     });
-    if (options.sttLifecycle === "turn-scoped") {
+    if (options.stt && options.sttLifecycle === "turn-scoped") {
       await closeAdapter("turn-commit");
     }
     await completeTurn(updatedSession, turn);
@@ -9600,7 +9653,7 @@ var runVoiceTelephonyBenchmark = async (scenarios = getDefaultVoiceTelephonyBenc
   };
 };
 // src/testing/tts.ts
-var DEFAULT_REALTIME_FORMAT = {
+var DEFAULT_REALTIME_FORMAT2 = {
   channels: 1,
   container: "raw",
   encoding: "pcm_s16le",
@@ -9659,7 +9712,7 @@ var runTTSAdapterFixture = async (adapter, fixture, options = {}) => {
   let audioDurationMs = 0;
   let audioChunkCount = 0;
   const session = adapter.kind === "realtime" ? await adapter.open({
-    format: options.realtimeFormat ?? DEFAULT_REALTIME_FORMAT,
+    format: options.realtimeFormat ?? DEFAULT_REALTIME_FORMAT2,
     sessionId: `tts-benchmark:${fixture.id}`,
     ...openOptions ?? {}
   }) : await adapter.open({

package/dist/types.d.ts

@@ -616,9 +616,11 @@ export type VoicePluginConfig<TContext = unknown, TSession extends VoiceSessionR
     lexicon?: VoiceLexiconEntry[] | VoiceLexiconResolver<TContext>;
     phraseHints?: VoicePhraseHint[] | VoicePhraseHintResolver<TContext>;
     preset?: VoiceRuntimePreset;
-    stt: STTAdapter;
+    stt?: STTAdapter;
     sttFallback?: VoiceSTTFallbackConfig;
     sttLifecycle?: VoiceSTTLifecycle;
+    realtime?: RealtimeAdapter;
+    realtimeInputFormat?: AudioFormat;
     tts?: TTSAdapter;
     session: VoiceSessionStore<NoInfer<TSession>>;
     reconnect?: VoiceReconnectConfig;
@@ -635,7 +637,9 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     id: string;
     context: TContext;
     socket: VoiceSocket;
-    stt: STTAdapter;
+    stt?: STTAdapter;
+    realtime?: RealtimeAdapter;
+    realtimeInputFormat?: AudioFormat;
     tts?: TTSAdapter;
     languageStrategy?: VoiceLanguageStrategy;
     lexicon?: VoiceLexiconEntry[];

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.127",
+	"version": "0.0.22-beta.128",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",