@cognidesk/voice-websocket 0.0.1

package/dist/index.d.ts

@@ -0,0 +1,375 @@
+import * as ws from 'ws';
+import { WebSocket } from 'ws';
+import { Server, IncomingMessage } from 'node:http';
+import { StartVoiceResult, RuntimeEvent, VoiceSocketMetadata, HandleVoiceUserMessageInput, HandleVoiceUserMessageResult, VoiceProfile } from '@cognidesk/core';
+
+declare const COGNIDESK_VOICE_PROTOCOL: "cognidesk.voice.v1";
+type VoiceProtocol = typeof COGNIDESK_VOICE_PROTOCOL;
+type VoiceBrowserClientEvent = {
+    type: "session.update";
+    event_id?: string;
+    session?: Record<string, unknown>;
+} | {
+    type: "input_audio_buffer.append";
+    event_id?: string;
+    audio: string;
+    sequence?: number;
+} | {
+    type: "input_audio_buffer.commit";
+    event_id?: string;
+} | {
+    type: "input_audio_buffer.clear";
+    event_id?: string;
+} | {
+    type: "response.cancel";
+    event_id?: string;
+    response_id?: string;
+    interruptedMessageId?: string;
+    playedUntilMs?: number;
+    audioEndMs?: number;
+    reason?: string;
+} | {
+    type: "conversation.item.truncate";
+    event_id?: string;
+    item_id?: string;
+    content_index?: number;
+    audio_end_ms?: number;
+};
+type VoiceBrowserServerEvent = {
+    type: "cognidesk.connection.ready";
+    event_id: string;
+    protocol: VoiceProtocol;
+    conversation: StartVoiceResult["conversation"];
+    channelSegment: StartVoiceResult["channelSegment"];
+    connection: StartVoiceResult["connection"];
+    lastAckSequence: number;
+} | {
+    type: "cognidesk.connection.reconnect_token";
+    event_id: string;
+    token: string;
+    expiresAt: string;
+} | {
+    type: "cognidesk.audio.ack";
+    event_id: string;
+    sequence: number;
+} | {
+    type: "cognidesk.runtime_event";
+    event_id: string;
+    event: RuntimeEvent;
+} | {
+    type: "cognidesk.turn.completed";
+    event_id: string;
+    text: string;
+    activeJourneyId?: string;
+} | {
+    type: "cognidesk.interruption.recorded";
+    event_id: string;
+    event: RuntimeEvent;
+} | {
+    type: "cognidesk.voice.preamble";
+    event_id: string;
+    text: string;
+    elapsedMs: number;
+} | {
+    type: "input_audio_buffer.speech_started";
+    event_id?: string;
+    audio_start_ms?: number;
+    item_id?: string;
+} | {
+    type: "input_audio_buffer.speech_stopped";
+    event_id?: string;
+    audio_end_ms?: number;
+    item_id?: string;
+} | {
+    type: "input_audio_transcription.completed";
+    event_id: string;
+    text: string;
+    item_id?: string;
+    startedAtMs?: number;
+    endedAtMs?: number;
+    transcriptionSource?: string;
+    metadata?: Record<string, unknown>;
+} | {
+    type: "response.output_audio.delta";
+    event_id?: string;
+    response_id?: string;
+    item_id?: string;
+    output_index?: number;
+    content_index?: number;
+    delta: string;
+} | {
+    type: "response.output_audio.done";
+    event_id?: string;
+    response_id?: string;
+    item_id?: string;
+    output_index?: number;
+    content_index?: number;
+} | {
+    type: "response.output_audio_transcript.delta";
+    event_id?: string;
+    response_id?: string;
+    item_id?: string;
+    output_index?: number;
+    content_index?: number;
+    delta: string;
+} | {
+    type: "response.output_audio_transcript.done";
+    event_id?: string;
+    response_id?: string;
+    item_id?: string;
+    output_index?: number;
+    content_index?: number;
+    transcript: string;
+} | {
+    type: "response.done";
+    event_id?: string;
+    response?: unknown;
+} | {
+    type: "error";
+    event_id?: string;
+    error: {
+        code: string;
+        message: string;
+        retryable?: boolean;
+        details?: unknown;
+    };
+};
+type VoiceProviderEvent = {
+    kind: "input_transcript.completed";
+    text: string;
+    itemId?: string;
+    startedAtMs?: number;
+    endedAtMs?: number;
+    transcriptionSource?: string;
+    metadata?: Record<string, unknown>;
+} | {
+    kind: "server_event";
+    event: VoiceBrowserServerEvent;
+} | {
+    kind: "runtime_events";
+    events: RuntimeEvent[];
+} | {
+    kind: "error";
+    code?: string;
+    message: string;
+    retryable?: boolean;
+    details?: unknown;
+};
+interface VoiceProviderConnectInput {
+    session: VoiceSocketSession;
+    profile?: VoiceProfile;
+    control?: VoiceControlSurface;
+    signal: AbortSignal;
+    onEvent(event: VoiceProviderEvent): Promise<void> | void;
+}
+interface VoiceProviderSession {
+    send(event: VoiceBrowserClientEvent): Promise<void> | void;
+    speak(input: {
+        text: string;
+        result?: HandleVoiceUserMessageResult;
+    }): Promise<void> | void;
+    preamble?(input: {
+        text: string;
+    }): Promise<void> | void;
+    close(): Promise<void> | void;
+}
+interface VoiceProvider {
+    readonly id: string;
+    connect(input: VoiceProviderConnectInput): Promise<VoiceProviderSession>;
+}
+interface VoiceControlTool {
+    name: string;
+    description?: string;
+    parameters?: unknown;
+}
+interface VoiceControlToolCall {
+    session: VoiceSocketSession;
+    name: string;
+    arguments: unknown;
+    callId: string;
+    itemId?: string;
+    responseId?: string;
+    signal: AbortSignal;
+    notify?(notification: VoiceControlNotification): Promise<void>;
+}
+interface VoiceControlToolResult {
+    output: unknown;
+    events?: RuntimeEvent[];
+}
+interface VoiceControlNotification {
+    message: string;
+    events?: RuntimeEvent[];
+    responseInstructions?: string;
+    createResponse?: boolean;
+}
+interface VoiceControlSurface {
+    tools: VoiceControlTool[];
+    instructions?: string;
+    createSessionInstructions?(input: {
+        session: VoiceSocketSession;
+    }): Promise<string> | string;
+    handleToolCall(input: VoiceControlToolCall): Promise<VoiceControlToolResult> | VoiceControlToolResult;
+}
+interface VoiceSocketLike {
+    send(data: string): void;
+    close(code?: number, reason?: string): void;
+    on(event: "message", listener: (data: string | ArrayBuffer | Uint8Array) => void): void;
+    on(event: "close", listener: (code?: number, reason?: string) => void): void;
+    on(event: "error", listener: (error: unknown) => void): void;
+}
+interface VoiceRecorder {
+    onAudio?(input: {
+        session: VoiceSocketSession;
+        speaker: "user" | "assistant";
+        audio: string;
+        sequence?: number;
+    }): Promise<void> | void;
+    onTranscript?(input: {
+        session: VoiceSocketSession;
+        speaker: "user" | "assistant";
+        text: string;
+        runtimeEvent?: RuntimeEvent;
+    }): Promise<void> | void;
+}
+interface VoiceSocketSession {
+    id: string;
+    conversation: StartVoiceResult["conversation"];
+    channelSegment: StartVoiceResult["channelSegment"];
+    connection: StartVoiceResult["connection"];
+    events: RuntimeEvent[];
+    createdAt: string;
+    updatedAt: string;
+    status: "pending" | "connected" | "reconnecting" | "ended";
+    lastAckSequence: number;
+    reconnectGraceUntil?: string;
+}
+interface VoiceSocketToken {
+    token: string;
+    connectionId: string;
+    sessionId: string;
+    purpose: "start" | "reconnect";
+    expiresAt: string;
+    consumedAt?: string;
+}
+interface VoiceSessionStore {
+    createSession(input: {
+        result: StartVoiceResult;
+        tokenTtlMs: number;
+        now?: Date;
+    }): Promise<{
+        session: VoiceSocketSession;
+        socket: VoiceSocketMetadata;
+    }>;
+    claimToken(input: {
+        connectionId: string;
+        token: string;
+        now?: Date;
+    }): Promise<{
+        session: VoiceSocketSession;
+        token: VoiceSocketToken;
+        reconnect: boolean;
+    } | null>;
+    issueReconnectToken(input: {
+        sessionId: string;
+        ttlMs: number;
+        now?: Date;
+    }): Promise<VoiceSocketToken>;
+    acknowledgeAudio(input: {
+        sessionId: string;
+        sequence: number;
+        now?: Date;
+    }): Promise<VoiceSocketSession>;
+    markConnected(sessionId: string, now?: Date): Promise<VoiceSocketSession>;
+    markReconnecting(sessionId: string, now?: Date, graceMs?: number): Promise<VoiceSocketSession>;
+    markEnded(sessionId: string, now?: Date): Promise<VoiceSocketSession>;
+    getSession(sessionId: string): Promise<VoiceSocketSession | null>;
+}
+interface InMemoryVoiceSessionStoreOptions {
+    createToken?: () => string;
+}
+declare function createInMemoryVoiceSessionStore(options?: InMemoryVoiceSessionStoreOptions): VoiceSessionStore;
+interface VoiceSocketHandshakeOptions {
+    store: VoiceSessionStore;
+    tokenTtlMs?: number;
+    pathPrefix?: string;
+    baseUrl?: string;
+}
+declare function createVoiceSocketHandshake(options: VoiceSocketHandshakeOptions): {
+    createSocket(input: {
+        result: StartVoiceResult;
+        request: Request;
+        basePath: string;
+    }): Promise<VoiceSocketMetadata>;
+};
+interface VoiceRuntime {
+    handleVoiceUserMessage<TTurn = unknown>(input: HandleVoiceUserMessageInput<TTurn>): Promise<HandleVoiceUserMessageResult>;
+    commitVoiceTranscript?(input: {
+        conversationId: string;
+        channelSegmentId: string;
+        speaker: "user" | "assistant";
+        text: string;
+        recordingReferenceId?: string;
+        startedAtMs?: number;
+        endedAtMs?: number;
+        transcriptionSource?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<{
+        events: RuntimeEvent[];
+        event: RuntimeEvent;
+        message: RuntimeEvent;
+    }>;
+    recordVoiceInterruption(input: {
+        conversationId: string;
+        channelSegmentId: string;
+        connectionId?: string;
+        interruptedMessageId?: string;
+        source?: "userSpeech" | "adapter" | "provider";
+        reason?: string;
+        recordingReferenceId?: string;
+        offsetMs?: number;
+    }): Promise<RuntimeEvent>;
+    endVoiceSegment(input: {
+        conversationId: string;
+        channelSegmentId: string;
+        connectionId?: string;
+        reason?: string;
+    }): Promise<RuntimeEvent>;
+}
+interface HandleVoiceSocketOptions {
+    socket: VoiceSocketLike;
+    connectionId: string;
+    token: string;
+    store: VoiceSessionStore;
+    runtime: VoiceRuntime;
+    provider: VoiceProvider;
+    control?: VoiceControlSurface;
+    profile?: VoiceProfile;
+    recorder?: VoiceRecorder;
+    initialGreeting?: string;
+    reconnectTokenTtlMs?: number;
+    reconnectGraceMs?: number;
+    inputTranscriptDebounceMs?: number;
+    turnPreambleMs?: number;
+    signal?: AbortSignal;
+}
+declare function handleVoiceSocket(options: HandleVoiceSocketOptions): Promise<void>;
+interface AttachNodeVoiceWebSocketAdapterOptions {
+    server: Server;
+    store: VoiceSessionStore;
+    runtime: VoiceRuntime;
+    provider: VoiceProvider;
+    control?: VoiceControlSurface;
+    profile?: VoiceProfile;
+    recorder?: VoiceRecorder;
+    pathPrefix?: string;
+    initialGreeting?: string;
+    reconnectTokenTtlMs?: number;
+    reconnectGraceMs?: number;
+    turnPreambleMs?: number;
+}
+declare function attachNodeVoiceWebSocketAdapter(options: AttachNodeVoiceWebSocketAdapterOptions): {
+    close(): void;
+    webSocketServer: ws.Server<typeof WebSocket, typeof IncomingMessage>;
+};
+
+export { type AttachNodeVoiceWebSocketAdapterOptions, COGNIDESK_VOICE_PROTOCOL, type HandleVoiceSocketOptions, type InMemoryVoiceSessionStoreOptions, type VoiceBrowserClientEvent, type VoiceBrowserServerEvent, type VoiceControlNotification, type VoiceControlSurface, type VoiceControlTool, type VoiceControlToolCall, type VoiceControlToolResult, type VoiceProtocol, type VoiceProvider, type VoiceProviderConnectInput, type VoiceProviderEvent, type VoiceProviderSession, type VoiceRecorder, type VoiceRuntime, type VoiceSessionStore, type VoiceSocketHandshakeOptions, type VoiceSocketLike, type VoiceSocketSession, type VoiceSocketToken, attachNodeVoiceWebSocketAdapter, createInMemoryVoiceSessionStore, createVoiceSocketHandshake, handleVoiceSocket };