@absolutejs/voice 0.0.22-beta.127 → 0.0.22-beta.129

package/dist/openaiRealtime.d.ts

@@ -0,0 +1,27 @@
+import type { RealtimeAdapter } from './types';
+export type OpenAIRealtimeModel = 'gpt-realtime' | 'gpt-realtime-mini' | 'gpt-4o-realtime-preview' | 'gpt-4o-mini-realtime-preview' | (string & {});
+export type OpenAIRealtimeVoice = 'alloy' | 'ash' | 'ballad' | 'cedar' | 'coral' | 'echo' | 'marin' | 'sage' | 'shimmer' | 'verse' | {
+    id: string;
+} | (string & {});
+export type OpenAIRealtimeTranscriptionModel = 'gpt-4o-mini-transcribe' | 'gpt-4o-transcribe' | 'whisper-1' | (string & {});
+export type OpenAIRealtimeNoiseReduction = 'near_field' | 'far_field';
+export type OpenAIRealtimeResponseMode = 'audio' | 'text';
+export type OpenAIRealtimeAdapterOptions = {
+    apiKey: string;
+    autoCommitSilenceMs?: number;
+    baseUrl?: string;
+    emitResponseTranscripts?: boolean;
+    inputTranscriptionLanguage?: string;
+    inputTranscriptionModel?: OpenAIRealtimeTranscriptionModel | null;
+    inputTranscriptionPrompt?: string;
+    instructions?: string;
+    maxOutputTokens?: number | 'inf';
+    model?: OpenAIRealtimeModel;
+    noiseReduction?: OpenAIRealtimeNoiseReduction;
+    responseMode?: OpenAIRealtimeResponseMode;
+    speed?: number;
+    temperature?: number;
+    voice?: OpenAIRealtimeVoice;
+    webSocket?: typeof WebSocket;
+};
+export declare const createOpenAIRealtimeAdapter: (options: OpenAIRealtimeAdapterOptions) => RealtimeAdapter;

package/dist/react/index.js

@@ -2567,6 +2567,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -2631,6 +2642,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -2895,6 +2907,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,

package/dist/svelte/index.js

@@ -865,6 +865,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -929,6 +940,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -1193,6 +1205,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,

package/dist/telephony/twilio.d.ts

@@ -2,7 +2,7 @@ import { Elysia } from 'elysia';
 import type { VoiceTelephonySetupStatus, VoiceTelephonySmokeCheck, VoiceTelephonySmokeReport } from './contract';
 import { type VoiceTelephonyOutcomePolicy, type VoiceTelephonyWebhookRoutesOptions } from '../telephonyOutcome';
 import { type VoiceCallReviewArtifact, type VoiceCallReviewConfig } from '../testing/review';
-import type { AudioFormat, VoiceLogger, VoicePluginConfig, VoiceSessionRecord, VoiceServerMessage } from '../types';
+import type { AudioFormat, STTAdapter, VoiceLogger, VoicePluginConfig, VoiceSessionRecord, VoiceServerMessage } from '../types';
 type TwilioMediaPayload = {
     chunk?: string;
     payload: string;
@@ -78,7 +78,7 @@ export type TwilioMediaStreamSocket = {
     close: (code?: number, reason?: string) => void | Promise<void>;
     send: (data: string) => void | Promise<void>;
 };
-export type TwilioMediaStreamBridgeOptions<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = Omit<VoicePluginConfig<TContext, TSession, TResult>, 'htmx' | 'path'> & {
+export type TwilioMediaStreamBridgeOptions<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = Omit<VoicePluginConfig<TContext, TSession, TResult>, 'htmx' | 'path' | 'stt'> & {
     clearOnInboundMedia?: boolean;
     context: TContext;
     logger?: VoiceLogger;
@@ -97,6 +97,7 @@ export type TwilioMediaStreamBridgeOptions<TContext = unknown, TSession extends
     };
     scenarioId?: string;
     sessionId?: string;
+    stt: STTAdapter;
 };
 export type TwilioMediaStreamBridge = {
     close: (reason?: string) => Promise<void>;

package/dist/testing/index.js

@@ -2126,6 +2126,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -2190,6 +2201,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -2454,6 +2466,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,
@@ -5033,6 +5059,12 @@ var DEFAULT_FORMAT = {
   encoding: "pcm_s16le",
   sampleRateHz: 16000
 };
+var DEFAULT_REALTIME_FORMAT = {
+  channels: 1,
+  container: "raw",
+  encoding: "pcm_s16le",
+  sampleRateHz: 24000
+};
 var toError = (value) => value instanceof Error ? value : new Error(String(value));
 var createEmptyCurrentTurn = () => ({
   finalText: "",
@@ -5310,6 +5342,18 @@ var createVoiceSession = (options) => {
       type: "call_lifecycle"
     });
   };
+  const sendReplay = async (session) => {
+    await send({
+      assistantTexts: session.turns.flatMap((turn) => turn.assistantText ? [turn.assistantText] : []),
+      call: session.call,
+      partial: session.currentTurn.partialText,
+      scenarioId: session.scenarioId,
+      sessionId: options.id,
+      status: session.status,
+      turns: session.turns,
+      type: "replay"
+    });
+  };
   const runHandoff = async (input) => {
     const queuedDelivery = options.handoff?.deliveryQueue ? createVoiceHandoffDeliveryRecord({
       action: input.action,
@@ -5413,6 +5457,23 @@ var createVoiceSession = (options) => {
       });
     }
   };
+  const sendAssistantAudio = async (chunk, input) => {
+    const normalizedChunk = chunk instanceof Uint8Array ? new Uint8Array(chunk) : chunk instanceof ArrayBuffer ? new Uint8Array(chunk.slice(0)) : new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
+    await send({
+      chunkBase64: encodeBase64(normalizedChunk),
+      format: input.format,
+      receivedAt: input.receivedAt,
+      turnId: activeTTSTurnId,
+      type: "audio"
+    });
+    if (activeTTSTurnId) {
+      await appendTurnLatencyStage({
+        at: input.receivedAt,
+        stage: "assistant_audio_received",
+        turnId: activeTTSTurnId
+      });
+    }
+  };
   const scheduleTurnCommit = (delayMs, reason, reset = true) => {
     if (!reset && silenceTimer) {
       return;
@@ -6114,8 +6175,12 @@ var createVoiceSession = (options) => {
     if (sttSession) {
       return sttSession;
     }
-    const openedSession = await options.stt.open({
-      format: DEFAULT_FORMAT,
+    const inputAdapter = options.realtime ?? options.stt;
+    if (!inputAdapter) {
+      throw new Error("Voice session requires either an stt or realtime adapter.");
+    }
+    const openedSession = await inputAdapter.open({
+      format: options.realtime ? options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT : DEFAULT_FORMAT,
       languageStrategy: options.languageStrategy,
       lexicon,
       phraseHints,
@@ -6150,6 +6215,16 @@ var createVoiceSession = (options) => {
     openedSession.on("close", (event) => {
       runAdapterEvent("adapter.close", () => handleClose(event));
     });
+    if (options.realtime) {
+      openedSession.on("audio", ({ chunk, format, receivedAt }) => {
+        runAdapterEvent("adapter.audio", async () => {
+          await sendAssistantAudio(chunk, {
+            format,
+            receivedAt
+          });
+        });
+      });
+    }
     return openedSession;
   };
   const ensureTTSSession = async () => {
@@ -6174,21 +6249,10 @@ var createVoiceSession = (options) => {
           if (ttsSession !== openedSession) {
             return;
           }
-          const normalizedChunk = chunk instanceof Uint8Array ? new Uint8Array(chunk) : chunk instanceof ArrayBuffer ? new Uint8Array(chunk.slice(0)) : new Uint8Array(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
-          await send({
-            chunkBase64: encodeBase64(normalizedChunk),
+          await sendAssistantAudio(chunk, {
             format,
-            receivedAt,
-            turnId: activeTTSTurnId,
-            type: "audio"
+            receivedAt
           });
-          if (activeTTSTurnId) {
-            await appendTurnLatencyStage({
-              at: receivedAt,
-              stage: "assistant_audio_received",
-              turnId: activeTTSTurnId
-            });
-          }
         });
       });
       openedSession.on("error", (event) => {
@@ -6267,7 +6331,8 @@ var createVoiceSession = (options) => {
       await appendTrace({
         payload: {
           text: output.assistantText,
-          ttsConfigured: Boolean(options.tts)
+          ttsConfigured: Boolean(options.tts),
+          realtimeConfigured: Boolean(options.realtime)
         },
         session,
         turnId: turn.id,
@@ -6299,9 +6364,35 @@ var createVoiceSession = (options) => {
             turnId: turn.id,
             type: "turn.assistant"
           });
+        } else if (options.realtime) {
+          const activeRealtimeSession = await ensureAdapter();
+          const realtimeStartedAt = Date.now();
+          activeTTSTurnId = turn.id;
+          await appendTurnLatencyStage({
+            at: realtimeStartedAt,
+            session,
+            stage: "tts_send_started",
+            turnId: turn.id
+          });
+          await activeRealtimeSession.send(output.assistantText);
+          await appendTurnLatencyStage({
+            session,
+            stage: "tts_send_completed",
+            turnId: turn.id
+          });
+          await appendTrace({
+            payload: {
+              elapsedMs: Date.now() - realtimeStartedAt,
+              mode: "realtime",
+              status: "sent"
+            },
+            session,
+            turnId: turn.id,
+            type: "turn.assistant"
+          });
         }
       } catch (error) {
-        logger.warn("voice tts send failed", {
+        logger.warn("voice assistant audio send failed", {
           error: toError(error).message,
           sessionId: options.id,
           turnId: turn.id
@@ -6309,7 +6400,7 @@ var createVoiceSession = (options) => {
         await appendTrace({
           payload: {
             error: toError(error).message,
-            status: "tts-send-failed"
+            status: options.realtime ? "realtime-send-failed" : "tts-send-failed"
           },
           session,
           turnId: turn.id,
@@ -6514,7 +6605,7 @@ var createVoiceSession = (options) => {
       turn,
       type: "turn"
     });
-    if (options.sttLifecycle === "turn-scoped") {
+    if (options.stt && options.sttLifecycle === "turn-scoped") {
       await closeAdapter("turn-commit");
     }
     await completeTurn(updatedSession, turn);
@@ -6577,6 +6668,7 @@ var createVoiceSession = (options) => {
       scenarioId: session.scenarioId,
       type: "session"
     });
+    await sendReplay(session);
     if (shouldFireOnSession) {
       await options.route.onCallStart?.({
         api,
@@ -9600,7 +9692,7 @@ var runVoiceTelephonyBenchmark = async (scenarios = getDefaultVoiceTelephonyBenc
   };
 };
 // src/testing/tts.ts
-var DEFAULT_REALTIME_FORMAT = {
+var DEFAULT_REALTIME_FORMAT2 = {
   channels: 1,
   container: "raw",
   encoding: "pcm_s16le",
@@ -9659,7 +9751,7 @@ var runTTSAdapterFixture = async (adapter, fixture, options = {}) => {
   let audioDurationMs = 0;
   let audioChunkCount = 0;
   const session = adapter.kind === "realtime" ? await adapter.open({
-    format: options.realtimeFormat ?? DEFAULT_REALTIME_FORMAT,
+    format: options.realtimeFormat ?? DEFAULT_REALTIME_FORMAT2,
     sessionId: `tts-benchmark:${fixture.id}`,
     ...openOptions ?? {}
   }) : await adapter.open({

package/dist/types.d.ts

@@ -616,9 +616,11 @@ export type VoicePluginConfig<TContext = unknown, TSession extends VoiceSessionR
     lexicon?: VoiceLexiconEntry[] | VoiceLexiconResolver<TContext>;
     phraseHints?: VoicePhraseHint[] | VoicePhraseHintResolver<TContext>;
     preset?: VoiceRuntimePreset;
-    stt: STTAdapter;
+    stt?: STTAdapter;
     sttFallback?: VoiceSTTFallbackConfig;
     sttLifecycle?: VoiceSTTLifecycle;
+    realtime?: RealtimeAdapter;
+    realtimeInputFormat?: AudioFormat;
     tts?: TTSAdapter;
     session: VoiceSessionStore<NoInfer<TSession>>;
     reconnect?: VoiceReconnectConfig;
@@ -635,7 +637,9 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     id: string;
     context: TContext;
     socket: VoiceSocket;
-    stt: STTAdapter;
+    stt?: STTAdapter;
+    realtime?: RealtimeAdapter;
+    realtimeInputFormat?: AudioFormat;
     tts?: TTSAdapter;
     languageStrategy?: VoiceLanguageStrategy;
     lexicon?: VoiceLexiconEntry[];
@@ -682,6 +686,16 @@ export type VoiceServerSessionMessage = {
     status: VoiceSessionStatus;
     scenarioId?: string;
 };
+export type VoiceServerReplayMessage<TResult = unknown> = {
+    type: 'replay';
+    assistantTexts: string[];
+    call?: VoiceCallLifecycleState;
+    partial: string;
+    scenarioId?: string;
+    sessionId: string;
+    status: VoiceSessionStatus;
+    turns: VoiceTurnRecord<TResult>[];
+};
 export type VoiceServerPartialMessage = {
     type: 'partial';
     transcript: Transcript;
@@ -723,7 +737,7 @@ export type VoiceServerErrorMessage = {
 export type VoiceServerPongMessage = {
     type: 'pong';
 };
-export type VoiceServerMessage<TResult = unknown> = VoiceServerSessionMessage | VoiceServerPartialMessage | VoiceServerFinalMessage | VoiceServerTurnMessage<TResult> | VoiceServerAssistantMessage | VoiceServerAudioMessage | VoiceServerCallLifecycleMessage | VoiceServerCompleteMessage | VoiceServerErrorMessage | VoiceServerPongMessage;
+export type VoiceServerMessage<TResult = unknown> = VoiceServerSessionMessage | VoiceServerReplayMessage<TResult> | VoiceServerPartialMessage | VoiceServerFinalMessage | VoiceServerTurnMessage<TResult> | VoiceServerAssistantMessage | VoiceServerAudioMessage | VoiceServerCallLifecycleMessage | VoiceServerCompleteMessage | VoiceServerErrorMessage | VoiceServerPongMessage;
 export type VoiceConnectionOptions = {
     protocols?: string[];
     scenarioId?: string;
@@ -974,6 +988,15 @@ export type VoiceStoreAction<TResult = unknown> = {
     sessionId: string;
     scenarioId?: string;
     status: VoiceSessionStatus;
+} | {
+    type: 'replay';
+    assistantTexts: string[];
+    call?: VoiceCallLifecycleState;
+    partial: string;
+    scenarioId?: string;
+    sessionId: string;
+    status: VoiceSessionStatus;
+    turns: VoiceTurnRecord<TResult>[];
 } | {
     type: 'call_lifecycle';
     event: VoiceCallLifecycleEvent;

package/dist/vue/index.js

@@ -2338,6 +2338,17 @@ var serverMessageToAction = (message) => {
         transcript: message.transcript,
         type: "partial"
       };
+    case "replay":
+      return {
+        assistantTexts: message.assistantTexts,
+        call: message.call,
+        partial: message.partial,
+        scenarioId: message.scenarioId,
+        sessionId: message.sessionId,
+        status: message.status,
+        turns: message.turns,
+        type: "replay"
+      };
     case "session":
       return {
         sessionId: message.sessionId,
@@ -2402,6 +2413,7 @@ var isVoiceServerMessage = (value) => {
     case "final":
     case "partial":
     case "pong":
+    case "replay":
     case "session":
     case "turn":
       return true;
@@ -2666,6 +2678,20 @@ var createVoiceStreamStore = () => {
           partial: action.transcript.text
         };
         break;
+      case "replay":
+        state = {
+          ...state,
+          assistantTexts: [...action.assistantTexts],
+          call: action.call ?? null,
+          error: null,
+          isConnected: action.status === "active",
+          partial: action.partial,
+          scenarioId: action.scenarioId ?? state.scenarioId,
+          sessionId: action.sessionId,
+          status: action.status,
+          turns: [...action.turns]
+        };
+        break;
       case "session":
         state = {
           ...state,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.127",
+	"version": "0.0.22-beta.129",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",