@absolutejs/voice 0.0.22-beta.127 → 0.0.22-beta.129

package/README.md

@@ -1324,6 +1324,59 @@ app.use(
 
 Client state now exposes `assistantAudio` on the stream/controller helpers, so apps can buffer or play synthesized chunks without inventing a second transport.
 
+## OpenAI Realtime
+
+Use `createOpenAIRealtimeAdapter(...)` when you want a direct OpenAI Realtime speech-to-speech output path for live smoke tests, duplex benchmarks, or custom realtime orchestration. It implements the same `RealtimeAdapter` contract used by the benchmark harness, so the provider can stream `response.output_audio.delta` audio chunks into AbsoluteJS voice events while still emitting normalized transcript, error, and close events.
+
+```ts
+import { createOpenAIRealtimeAdapter } from '@absolutejs/voice';
+import { runTTSAdapterFixture } from '@absolutejs/voice/testing';
+
+const realtime = createOpenAIRealtimeAdapter({
+	apiKey: process.env.OPENAI_API_KEY!,
+	instructions: 'Answer in one concise sentence.',
+	model: 'gpt-realtime',
+	voice: 'marin'
+});
+
+app.use(
+	voice({
+		path: '/voice',
+		realtime,
+		realtimeInputFormat: {
+			channels: 1,
+			container: 'raw',
+			encoding: 'pcm_s16le',
+			sampleRateHz: 24000
+		},
+		session,
+		onTurn: async ({ turn }) => ({
+			assistantText: `You said: ${turn.text}`
+		}),
+		onComplete: async () => {}
+	})
+);
+
+const report = await runTTSAdapterFixture(
+	realtime,
+	{
+		id: 'openai-realtime-smoke',
+		text: 'Say exactly: AbsoluteJS realtime is online.',
+		title: 'OpenAI Realtime smoke'
+	},
+	{
+		realtimeFormat: {
+			channels: 1,
+			container: 'raw',
+			encoding: 'pcm_s16le',
+			sampleRateHz: 24000
+		}
+	}
+);
+```
+
+For server-to-server use, the adapter opens a WebSocket to OpenAI, sends `session.update`, streams text or base64 PCM input, and emits raw 24kHz mono `pcm_s16le` assistant audio. It requires raw 24kHz mono PCM input because that is the OpenAI Realtime PCM format. The main `voice(...)` route can now run in cascaded mode with `stt` plus optional `tts`, or direct realtime mode with `realtime`. Browser demos should make sure the captured PCM format matches `realtimeInputFormat` or resample before sending audio.
+
 If you want a minimal browser playback path, use the client audio player:
 
 ```ts

package/dist/angular/index.js

@@ -430,6 +430,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -494,6 +505,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -758,6 +770,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,

package/dist/client/actions.d.ts

@@ -10,8 +10,12 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     event?: undefined;
     message?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     text: string;
@@ -24,8 +28,12 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     event?: undefined;
     message?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     sessionId: string;
@@ -38,8 +46,12 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     event?: undefined;
     message?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     event: import("..").VoiceCallLifecycleEvent;
@@ -52,8 +64,12 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     text?: undefined;
     message?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     message: string;
@@ -66,8 +82,12 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     sessionId?: undefined;
     event?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     transcript: import("..").Transcript;
@@ -80,8 +100,12 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     sessionId?: undefined;
     event?: undefined;
     message?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     transcript: import("..").Transcript;
@@ -94,8 +118,30 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     sessionId?: undefined;
     event?: undefined;
     message?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
+    turn?: undefined;
+} | {
+    assistantTexts: string[];
+    call: import("..").VoiceCallLifecycleState | undefined;
+    partial: string;
+    scenarioId: string | undefined;
+    sessionId: string;
+    status: import("..").VoiceSessionStatus;
+    turns: import("..").VoiceTurnRecord<TResult>[];
+    type: "replay";
+    chunk?: undefined;
+    format?: undefined;
+    receivedAt?: undefined;
+    turnId?: undefined;
+    text?: undefined;
+    event?: undefined;
+    message?: undefined;
+    transcript?: undefined;
     turn?: undefined;
 } | {
     sessionId: string;
@@ -110,6 +156,10 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     event?: undefined;
     message?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
+    turns?: undefined;
     turn?: undefined;
 } | {
     turn: import("..").VoiceTurnRecord<TResult>;
@@ -123,6 +173,10 @@ export declare const serverMessageToAction: <TResult = unknown>(message: VoiceSe
     event?: undefined;
     message?: undefined;
     transcript?: undefined;
+    assistantTexts?: undefined;
+    call?: undefined;
+    partial?: undefined;
     scenarioId?: undefined;
     status?: undefined;
+    turns?: undefined;
 } | null;

package/dist/client/htmxBootstrap.js

@@ -209,6 +209,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -273,6 +284,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -537,6 +549,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,

package/dist/client/index.js

@@ -116,6 +116,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -701,6 +702,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -811,6 +823,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,

package/dist/index.d.ts

@@ -31,6 +31,7 @@ export { createVoicePhoneAgent } from './phoneAgent';
 export { createStoredVoiceCallReviewArtifact, createStoredVoiceExternalObjectMap, createStoredVoiceIntegrationEvent, createStoredVoiceOpsTask, createVoiceFileExternalObjectMapStore, createVoiceFileAssistantMemoryStore, createVoiceFileAuditEventStore, createVoiceFileAuditSinkDeliveryStore, createVoiceFileCampaignStore, createVoiceFileIntegrationEventStore, createVoiceFileReviewStore, createVoiceFileRuntimeStorage, createVoiceFileSessionStore, createVoiceFileTaskStore, createVoiceFileTraceSinkDeliveryStore, createVoiceFileTraceEventStore } from './fileStore';
 export { createVoiceAssistantMemoryHandle, createVoiceAssistantMemoryRecord, createVoiceMemoryAssistantMemoryStore, resolveVoiceAssistantMemoryNamespace } from './assistantMemory';
 export { createAnthropicVoiceAssistantModel, createGeminiVoiceAssistantModel, createJSONVoiceAssistantModel, createOpenAIVoiceAssistantModel, resolveVoiceProviderRoutingPolicyPreset, createVoiceProviderRouter } from './modelAdapters';
+export { createOpenAIRealtimeAdapter } from './openaiRealtime';
 export { createOpenAIVoiceTTS } from './openaiTTS';
 export { createVoiceProviderHealthHTMLHandler, createVoiceProviderHealthJSONHandler, createVoiceProviderHealthRoutes, renderVoiceProviderHealthHTML, summarizeVoiceProviderHealth } from './providerHealth';
 export { createVoiceProviderCapabilityHTMLHandler, createVoiceProviderCapabilityJSONHandler, createVoiceProviderCapabilityRoutes, renderVoiceProviderCapabilityHTML, summarizeVoiceProviderCapabilities } from './providerCapabilities';
@@ -81,6 +82,7 @@ export type { VoiceWorkflowContract, VoiceWorkflowContractDefinition, VoiceWorkf
 export type { VoiceSessionListHTMLHandlerOptions, VoiceSessionListItem, VoiceSessionListOptions, VoiceSessionListRoutesOptions, VoiceSessionListStatus, VoiceSessionReplay, VoiceSessionReplayHTMLHandlerOptions, VoiceSessionReplayOptions, VoiceSessionReplayRoutesOptions, VoiceSessionReplayTurn } from './sessionReplay';
 export type { AnthropicVoiceAssistantModelOptions, GeminiVoiceAssistantModelOptions, OpenAIVoiceAssistantModelOptions, VoiceProviderRouterEvent, VoiceProviderRouterFallbackMode, VoiceProviderRouterHealthOptions, VoiceProviderRouterOptions, VoiceProviderRouterPolicy, VoiceProviderRouterPolicyPreset, VoiceProviderRouterPolicyWeights, VoiceProviderRouterProviderHealth, VoiceProviderRouterProviderProfile, VoiceProviderRouterStrategy, VoiceJSONAssistantModelHandler, VoiceJSONAssistantModelOptions } from './modelAdapters';
 export type { OpenAIVoiceTTSOptions, OpenAIVoiceTTSVoice } from './openaiTTS';
+export type { OpenAIRealtimeAdapterOptions, OpenAIRealtimeModel, OpenAIRealtimeNoiseReduction, OpenAIRealtimeResponseMode, OpenAIRealtimeTranscriptionModel, OpenAIRealtimeVoice } from './openaiRealtime';
 export type { VoiceProviderHealthStatus, VoiceProviderHealthSummary, VoiceProviderHealthSummaryOptions } from './providerHealth';
 export type { VoiceProviderCapabilityDefinition, VoiceProviderCapabilityHandlerOptions, VoiceProviderCapabilityHTMLHandlerOptions, VoiceProviderCapabilityKind, VoiceProviderCapabilityOptions, VoiceProviderCapabilityReport, VoiceProviderCapabilityRoutesOptions, VoiceProviderCapabilitySummary } from './providerCapabilities';
 export type { VoiceProviderRoutingContractDefinition, VoiceProviderRoutingContractIssue, VoiceProviderRoutingContractReport, VoiceProviderRoutingContractRunOptions, VoiceProviderRoutingExpectation, VoiceProviderRoutingStatus } from './providerRoutingContract';