@absolutejs/voice 0.0.22-beta.34 → 0.0.22-beta.341

package/dist/turnQuality.d.ts

@@ -0,0 +1,94 @@
+import { Elysia } from 'elysia';
+import type { VoiceSessionRecord, VoiceSessionStore, VoiceTranscriptQuality } from './types';
+export type VoiceTurnQualityStatus = 'pass' | 'warn' | 'fail' | 'unknown';
+export type VoiceTurnQualityItem = {
+    averageConfidence?: number;
+    committedAt: number;
+    correctionChanged: boolean;
+    correctionProvider?: string;
+    correctionReason?: string;
+    costUnits?: number;
+    fallbackSelectionReason?: string;
+    fallbackUsed: boolean;
+    finalTranscriptCount: number;
+    latencyMs?: number;
+    partialTranscriptCount: number;
+    selectedTranscriptCount: number;
+    sessionId: string;
+    source?: VoiceTranscriptQuality['source'];
+    status: VoiceTurnQualityStatus;
+    text: string;
+    turnId: string;
+};
+export type VoiceTurnQualityReport = {
+    checkedAt: number;
+    failed: number;
+    sessions: number;
+    status: VoiceTurnQualityStatus;
+    total: number;
+    turns: VoiceTurnQualityItem[];
+    warnings: number;
+};
+export type VoiceTurnQualityOptions<TSession extends VoiceSessionRecord = VoiceSessionRecord> = {
+    confidenceWarnThreshold?: number;
+    limit?: number;
+    sessionIds?: string[];
+    sessions?: TSession[];
+    store?: VoiceSessionStore<TSession>;
+};
+export type VoiceTurnQualityHTMLHandlerOptions<TSession extends VoiceSessionRecord = VoiceSessionRecord> = VoiceTurnQualityOptions<TSession> & {
+    headers?: HeadersInit;
+    render?: (report: VoiceTurnQualityReport) => string | Promise<string>;
+    title?: string;
+};
+export type VoiceTurnQualityRoutesOptions<TSession extends VoiceSessionRecord = VoiceSessionRecord> = VoiceTurnQualityHTMLHandlerOptions<TSession> & {
+    htmlPath?: false | string;
+    name?: string;
+    path?: string;
+};
+export declare const summarizeVoiceTurnQuality: <TSession extends VoiceSessionRecord = VoiceSessionRecord>(options: VoiceTurnQualityOptions<TSession>) => Promise<VoiceTurnQualityReport>;
+export declare const renderVoiceTurnQualityHTML: (report: VoiceTurnQualityReport, options?: {
+    title?: string;
+}) => string;
+export declare const createVoiceTurnQualityJSONHandler: <TSession extends VoiceSessionRecord = VoiceSessionRecord>(options: VoiceTurnQualityOptions<TSession>) => () => Promise<VoiceTurnQualityReport>;
+export declare const createVoiceTurnQualityHTMLHandler: <TSession extends VoiceSessionRecord = VoiceSessionRecord>(options: VoiceTurnQualityHTMLHandlerOptions<TSession>) => () => Promise<Response>;
+export declare const createVoiceTurnQualityRoutes: <TSession extends VoiceSessionRecord = VoiceSessionRecord>(options: VoiceTurnQualityRoutesOptions<TSession>) => Elysia<"", {
+    decorator: {};
+    store: {};
+    derive: {};
+    resolve: {};
+}, {
+    typebox: {};
+    error: {};
+}, {
+    schema: {};
+    standaloneSchema: {};
+    macro: {};
+    macroFn: {};
+    parser: {};
+    response: {};
+}, {
+    [x: string]: {
+        get: {
+            body: unknown;
+            params: {};
+            query: unknown;
+            headers: unknown;
+            response: {
+                200: VoiceTurnQualityReport;
+            };
+        };
+    };
+}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}>;

package/dist/types.d.ts

@@ -1,8 +1,10 @@
 import type { SessionStore } from '@absolutejs/absolute';
+import type { MediaWebRTCStatsCollector, MediaWebRTCStatsReport, MediaWebRTCStatsReportInput, MediaWebRTCStreamContinuityInput, MediaWebRTCStreamContinuityReport } from '@absolutejs/media';
 import type { VoiceOpsDispositionTaskPolicies, VoiceOpsTaskAssignmentRule, VoiceOpsTaskAssignmentRules, VoiceIntegrationWebhookConfig, StoredVoiceIntegrationEvent, StoredVoiceOpsTask, VoiceIntegrationEventStore, VoiceOpsTaskPolicy, VoiceOpsTask, VoiceOpsTaskStore } from './ops';
 import type { VoiceIntegrationSink } from './opsSinks';
 import type { StoredVoiceCallReviewArtifact, VoiceCallReviewArtifact, VoiceCallReviewStore } from './testing/review';
 import type { VoiceTraceEventStore } from './trace';
+import type { VoiceLiveOpsControlState } from './liveOps';
 export type AudioFormat = {
     container: 'raw';
     encoding: 'alaw' | 'mulaw' | 'pcm_s16le';
@@ -193,6 +195,15 @@ export type RealtimeAdapter<TOptions extends RealtimeAdapterOpenOptions = Realti
     open: (options: TOptions) => Promise<RealtimeAdapterSession> | RealtimeAdapterSession;
 };
 export type VoiceSessionStatus = 'active' | 'reconnecting' | 'completed' | 'failed';
+export type VoiceReconnectClientStatus = 'idle' | 'reconnecting' | 'resumed' | 'exhausted';
+export type VoiceReconnectClientState = {
+    attempts: number;
+    lastDisconnectAt?: number;
+    lastResumedAt?: number;
+    maxAttempts: number;
+    nextAttemptAt?: number;
+    status: VoiceReconnectClientStatus;
+};
 export type VoiceTurnRecord<TResult = unknown> = {
     id: string;
     text: string;
@@ -475,6 +486,10 @@ export type VoiceLexiconResolver<TContext = unknown> = (input: {
 }) => Promise<VoiceLexiconEntry[] | void> | VoiceLexiconEntry[] | void;
 export type VoiceOnTurnObjectHandler<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = (input: {
     context: TContext;
+    liveOps?: {
+        control: VoiceLiveOpsControlState;
+        injectedInstruction?: string;
+    };
     session: TSession;
     turn: VoiceTurnRecord;
     api: VoiceSessionHandle<TContext, TSession, TResult>;
@@ -596,6 +611,9 @@ export type VoiceRuntimeOpsConfig<TContext = unknown, TSession extends VoiceSess
     tasks?: VoiceOpsTaskStore;
     webhook?: VoiceIntegrationWebhookConfig;
 };
+export type VoiceLiveOpsRuntimeConfig = {
+    getControl: (sessionId: string) => Promise<VoiceLiveOpsControlState | null | undefined> | VoiceLiveOpsControlState | null | undefined;
+};
 export type VoiceNormalizedRouteConfig<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = Omit<VoiceRouteConfig<TContext, TSession, TResult>, 'onTurn'> & {
     onTurn: VoiceOnTurnObjectHandler<TContext, TSession, TResult>;
 };
@@ -616,9 +634,11 @@ export type VoicePluginConfig<TContext = unknown, TSession extends VoiceSessionR
     lexicon?: VoiceLexiconEntry[] | VoiceLexiconResolver<TContext>;
     phraseHints?: VoicePhraseHint[] | VoicePhraseHintResolver<TContext>;
     preset?: VoiceRuntimePreset;
-    stt: STTAdapter;
+    stt?: STTAdapter;
     sttFallback?: VoiceSTTFallbackConfig;
     sttLifecycle?: VoiceSTTLifecycle;
+    realtime?: RealtimeAdapter;
+    realtimeInputFormat?: AudioFormat;
     tts?: TTSAdapter;
     session: VoiceSessionStore<NoInfer<TSession>>;
     reconnect?: VoiceReconnectConfig;
@@ -628,6 +648,7 @@ export type VoicePluginConfig<TContext = unknown, TSession extends VoiceSessionR
     htmx?: boolean | VoiceHTMXConfig<TSession, NoInfer<TResult>>;
     handoff?: VoiceHandoffConfig<TContext, TSession, TResult>;
     ops?: VoiceRuntimeOpsConfig<TContext, TSession, TResult>;
+    liveOps?: VoiceLiveOpsRuntimeConfig;
     trace?: VoiceTraceEventStore;
 } & VoiceRouteConfig<TContext, TSession, TResult>;
 export type CreateVoiceSessionOptions<TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown> = {
@@ -635,7 +656,9 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     id: string;
     context: TContext;
     socket: VoiceSocket;
-    stt: STTAdapter;
+    stt?: STTAdapter;
+    realtime?: RealtimeAdapter;
+    realtimeInputFormat?: AudioFormat;
     tts?: TTSAdapter;
     languageStrategy?: VoiceLanguageStrategy;
     lexicon?: VoiceLexiconEntry[];
@@ -649,6 +672,7 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     turnDetection: VoiceResolvedTurnDetectionConfig;
     audioConditioning?: VoiceResolvedAudioConditioningConfig;
     handoff?: VoiceHandoffConfig<TContext, TSession, TResult>;
+    liveOps?: VoiceLiveOpsRuntimeConfig;
     route: VoiceNormalizedRouteConfig<TContext, TSession, TResult>;
     logger?: VoiceLogger;
 };
@@ -682,6 +706,16 @@ export type VoiceServerSessionMessage = {
     status: VoiceSessionStatus;
     scenarioId?: string;
 };
+export type VoiceServerReplayMessage<TResult = unknown> = {
+    type: 'replay';
+    assistantTexts: string[];
+    call?: VoiceCallLifecycleState;
+    partial: string;
+    scenarioId?: string;
+    sessionId: string;
+    status: VoiceSessionStatus;
+    turns: VoiceTurnRecord<TResult>[];
+};
 export type VoiceServerPartialMessage = {
     type: 'partial';
     transcript: Transcript;
@@ -723,17 +757,43 @@ export type VoiceServerErrorMessage = {
 export type VoiceServerPongMessage = {
     type: 'pong';
 };
-export type VoiceServerMessage<TResult = unknown> = VoiceServerSessionMessage | VoiceServerPartialMessage | VoiceServerFinalMessage | VoiceServerTurnMessage<TResult> | VoiceServerAssistantMessage | VoiceServerAudioMessage | VoiceServerCallLifecycleMessage | VoiceServerCompleteMessage | VoiceServerErrorMessage | VoiceServerPongMessage;
+export type VoiceServerConnectionMessage = {
+    type: 'connection';
+    reconnect: VoiceReconnectClientState;
+};
+export type VoiceServerMessage<TResult = unknown> = VoiceServerSessionMessage | VoiceServerReplayMessage<TResult> | VoiceServerPartialMessage | VoiceServerFinalMessage | VoiceServerTurnMessage<TResult> | VoiceServerAssistantMessage | VoiceServerAudioMessage | VoiceServerCallLifecycleMessage | VoiceServerCompleteMessage | VoiceServerErrorMessage | VoiceServerPongMessage | VoiceServerConnectionMessage;
 export type VoiceConnectionOptions = {
+    browserMedia?: false | VoiceBrowserMediaReporterOptions;
     protocols?: string[];
     scenarioId?: string;
     reconnect?: boolean;
+    reconnectReportPath?: string;
     maxReconnectAttempts?: number;
     pingInterval?: number;
     sessionId?: string;
 };
+export type VoiceBrowserMediaReportPayload = {
+    at: number;
+    continuity?: MediaWebRTCStreamContinuityReport;
+    report: MediaWebRTCStatsReport;
+    scenarioId?: string | null;
+    sessionId?: string | null;
+};
+export type VoiceBrowserMediaReporterOptions = Omit<MediaWebRTCStatsReportInput, 'peerConnection'> & {
+    fetch?: typeof fetch;
+    getPeerConnection?: (() => MediaWebRTCStatsCollector | null | undefined) | (() => Promise<MediaWebRTCStatsCollector | null | undefined>);
+    getScenarioId?: () => string | null | undefined;
+    getSessionId?: () => string | null | undefined;
+    intervalMs?: number;
+    continuity?: false | Omit<MediaWebRTCStreamContinuityInput, 'previousStats' | 'stats'>;
+    onError?: (error: unknown) => void;
+    onReport?: (payload: VoiceBrowserMediaReportPayload) => void;
+    path?: string;
+    peerConnection?: MediaWebRTCStatsCollector;
+};
 export type VoiceCaptureOptions = {
     channelCount?: 1 | 2;
+    onAudio?: (audio: Uint8Array | ArrayBuffer, sendAudio: (audio: Uint8Array | ArrayBuffer) => void) => void;
     onLevel?: (level: number) => void;
     sampleRateHz?: number;
 };
@@ -747,6 +807,46 @@ export type VoiceBargeInOptions = {
     enabled?: boolean;
     interruptOnPartial?: boolean;
     interruptThreshold?: number;
+    monitor?: VoiceBargeInMonitor;
+};
+export type VoiceBargeInTriggerReason = 'input-level' | 'manual-audio' | 'manual-interrupt' | 'partial-transcript';
+export type VoiceBargeInMonitorEvent = {
+    at: number;
+    id: string;
+    latencyMs?: number;
+    playbackStopLatencyMs?: number;
+    reason: VoiceBargeInTriggerReason;
+    sessionId?: string | null;
+    status: 'requested' | 'stopped' | 'skipped';
+    thresholdMs?: number;
+};
+export type VoiceBargeInMonitorSnapshot = {
+    averageLatencyMs?: number;
+    events: VoiceBargeInMonitorEvent[];
+    failed: number;
+    lastEvent?: VoiceBargeInMonitorEvent;
+    passed: number;
+    status: 'empty' | 'fail' | 'pass' | 'warn';
+    thresholdMs: number;
+    total: number;
+};
+export type VoiceBargeInMonitor = {
+    getSnapshot: () => VoiceBargeInMonitorSnapshot;
+    recordRequested: (input: {
+        reason: VoiceBargeInTriggerReason;
+        sessionId?: string | null;
+    }) => VoiceBargeInMonitorEvent;
+    recordSkipped: (input: {
+        reason: VoiceBargeInTriggerReason;
+        sessionId?: string | null;
+    }) => VoiceBargeInMonitorEvent;
+    recordStopped: (input: {
+        latencyMs?: number;
+        playbackStopLatencyMs?: number;
+        reason: VoiceBargeInTriggerReason;
+        sessionId?: string | null;
+    }) => VoiceBargeInMonitorEvent;
+    subscribe: (subscriber: () => void) => () => void;
 };
 export type VoiceAudioPlayerOptions = {
     autoStart?: boolean;
@@ -805,6 +905,7 @@ export type VoiceStreamState<TResult = unknown> = {
     sessionId: string | null;
     scenarioId: string | null;
     status: VoiceSessionStatus | 'idle';
+    reconnect: VoiceReconnectClientState;
     partial: string;
     turns: VoiceTurnRecord<TResult>[];
     assistantTexts: string[];
@@ -831,6 +932,7 @@ export type VoiceStream<TResult = unknown> = {
     getSnapshot: () => VoiceStreamState<TResult>;
     isConnected: boolean;
     partial: string;
+    reconnect: VoiceReconnectClientState;
     sendAudio: (audio: Uint8Array | ArrayBuffer) => void;
     sessionId: string | null;
     scenarioId: string | null;
@@ -900,6 +1002,7 @@ export type VoiceController<TResult = unknown> = {
     isConnected: boolean;
     isRecording: boolean;
     partial: string;
+    reconnect: VoiceReconnectClientState;
     recordingError: string | null;
     sendAudio: (audio: Uint8Array | ArrayBuffer) => void;
     sessionId: string | null;
@@ -933,6 +1036,15 @@ export type VoiceStoreAction<TResult = unknown> = {
     sessionId: string;
     scenarioId?: string;
     status: VoiceSessionStatus;
+} | {
+    type: 'replay';
+    assistantTexts: string[];
+    call?: VoiceCallLifecycleState;
+    partial: string;
+    scenarioId?: string;
+    sessionId: string;
+    status: VoiceSessionStatus;
+    turns: VoiceTurnRecord<TResult>[];
 } | {
     type: 'call_lifecycle';
     event: VoiceCallLifecycleEvent;
@@ -963,6 +1075,9 @@ export type VoiceStoreAction<TResult = unknown> = {
     message: string;
 } | {
     type: 'connected';
+} | {
+    type: 'connection';
+    reconnect: VoiceReconnectClientState;
 } | {
     type: 'disconnected';
 };