@byokit/realtime 0.1.0

package/dist/providers/openai.js

@@ -0,0 +1,313 @@
+import WebSocket from 'ws';
+import { responseFence } from "../policy.js";
+import { cleanProse, providerRefusal as refusal } from "../prose.js";
+// Lifted wire behavior; this module is loaded only by the kit child process.
+export function createAdapter(options) {
+    const MODEL = options.model ?? 'gpt-realtime-2.1';
+    const RATE = 24000;
+    const PROVIDER_URL = options.endpoint ?? `wss://api.openai.com/v1/realtime?model=${MODEL}`;
+    let endAfterResponse = false;
+    const providerTools = options.tools.map((tool) => ({ type: 'function', ...tool }));
+    const currentContext = '';
+    const isExplicitHangup = options.hangup ?? (() => false);
+    const cleanProviderProse = (value, fallback, max) => cleanProse(value, fallback, max, options.redact);
+    const providerRefusal = (status, body) => refusal(status, body, options.redact);
+    const tools = options.bridge;
+    const emit = (frame) => process.stdout.write(`${JSON.stringify(frame)}\n`);
+    // Provider deltas may exceed the public frame bound after tool calls. Base64 is
+    // independently decodable when split on four-character boundaries.
+    function chunkAudio(audio) {
+        const chunks = [];
+        const chunkSize = 64 * 1024;
+        for (let offset = 0; offset < audio.length; offset += chunkSize)
+            chunks.push(audio.slice(offset, offset + chunkSize));
+        return chunks;
+    }
+    const emitAudio = (audio) => {
+        for (const data of chunkAudio(audio))
+            emit({ type: 'realtime.audio', data });
+    };
+    const state = (value, detail) => tools.state(value, detail);
+    const PROMPT = options.instructions;
+    const fence = responseFence();
+    let reconnectTimer, stableTimer;
+    let providerReconnects = 0, responseActive = false;
+    const interrupt = () => { fence.interrupt(); responseActive = false; pendingToolCalls.clear(); emit({ type: 'realtime.audio.clear' }); };
+    let closing = false;
+    const close = (reason) => {
+        if (closing)
+            return;
+        closing = true;
+        stopped = true;
+        fence.interrupt();
+        clearTimeout(reconnectTimer);
+        clearTimeout(stableTimer);
+        tools.close();
+        process.stdout.write(`${JSON.stringify({ type: 'realtime.closed', reason })}\n`, () => process.exit(0));
+    };
+    let ws;
+    let stopped = false;
+    let providerReady = false;
+    const text = (value) => String(value ?? '').trim();
+    function providerError(error) {
+        const raw = typeof error === 'string' ? error : error?.message ?? error?.code ?? 'provider error';
+        const detail = cleanProviderProse(raw, 'provider error', 200);
+        return { detail, terminal: /api key|auth|credit|quota|billing|permission|forbidden|invalid json|invalid payload|unknown name|unsupported/i.test(detail) };
+    }
+    const runTool = (name, input, operationId, signal) => tools.run(name, input, operationId, signal);
+    const pendingToolCalls = new Map();
+    function handleOpenAiEvent(raw) {
+        let message;
+        try {
+            message = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        switch (message.type) {
+            case 'session.updated':
+                providerReady = true;
+                emit({ type: 'realtime.ready', inputRate: RATE, outputRate: RATE });
+                clearTimeout(stableTimer);
+                stableTimer = setTimeout(() => { providerReconnects = 0; }, 30000);
+                state('connected');
+                break;
+            case 'input_audio_buffer.speech_started':
+                interrupt();
+                break;
+            case 'response.created':
+                if (!fence.begin(message.response?.id))
+                    break;
+                responseActive = true;
+                state('thinking');
+                break;
+            case 'response.output_audio.delta':
+            case 'response.audio.delta': {
+                const audio = typeof message.delta === 'string' ? message.delta : typeof message.audio === 'string' ? message.audio : '';
+                if (audio && fence.accepts(message.response_id))
+                    emitAudio(audio);
+                break;
+            }
+            case 'response.done': {
+                if (message.response?.usage)
+                    emit({ type: 'realtime.usage', usage: { seconds: 0, basis: 'tokens', inputTokens: message.response.usage.input_tokens ?? 0, outputTokens: message.response.usage.output_tokens ?? 0, audioInTokens: message.response.usage.input_token_details?.audio_tokens ?? 0, audioOutTokens: message.response.usage.output_token_details?.audio_tokens ?? 0, cachedTokens: message.response.usage.input_token_details?.cached_tokens ?? 0 } });
+                if (!fence.accepts(message.response?.id))
+                    break;
+                responseActive = false;
+                const responseId = message.response?.id;
+                const calls = typeof responseId === 'string' ? pendingToolCalls.get(responseId) ?? [] : [];
+                if (typeof responseId === 'string')
+                    pendingToolCalls.delete(responseId);
+                if (message.response?.status === 'completed' && calls.length > 0) {
+                    const signal = fence.signal, current = ws;
+                    void (async () => {
+                        const outputs = [];
+                        for (const call of calls) {
+                            if (signal.aborted)
+                                return;
+                            let output;
+                            try {
+                                output = await runTool(call.name, call.args, call.callId, signal);
+                            }
+                            catch {
+                                output = 'The request could not be completed. Please try again.';
+                            }
+                            outputs.push({ callId: call.callId, output });
+                        }
+                        if (stopped || signal.aborted || ws !== current || ws?.readyState !== WebSocket.OPEN)
+                            return;
+                        for (const output of outputs) {
+                            ws.send(JSON.stringify({
+                                type: 'conversation.item.create',
+                                item: { type: 'function_call_output', call_id: output.callId, output: text(output.output).slice(0, 24000) },
+                            }));
+                        }
+                        ws.send(JSON.stringify({ type: 'response.create' }));
+                    })();
+                }
+                else if (message.response?.status === 'completed' && endAfterResponse) {
+                    stopped = true;
+                    ws?.close();
+                    close('ended');
+                }
+                else {
+                    state('connected');
+                }
+                break;
+            }
+            case 'conversation.item.input_audio_transcription.completed':
+                if (typeof message.transcript === 'string' && message.transcript.trim() !== '') {
+                    if (isExplicitHangup(message.transcript))
+                        endAfterResponse = true;
+                    emit({ type: 'realtime.transcript', role: 'user', text: message.transcript });
+                }
+                break;
+            case 'response.output_audio_transcript.done':
+                if (!fence.accepts(message.response_id))
+                    break;
+                if (typeof message.transcript === 'string' && message.transcript.trim() !== '') {
+                    tools.answered();
+                    emit({ type: 'realtime.transcript', role: 'agent', text: message.transcript });
+                }
+                break;
+            case 'response.function_call_arguments.done': {
+                if (!fence.accepts(message.response_id))
+                    break;
+                if (typeof message.response_id !== 'string')
+                    break;
+                let args = {};
+                try {
+                    args = JSON.parse(message.arguments || '{}');
+                }
+                catch { /* interrupted arguments */ }
+                const calls = pendingToolCalls.get(message.response_id) ?? [];
+                calls.push({ name: message.name, callId: message.call_id, args });
+                pendingToolCalls.set(message.response_id, calls);
+                break;
+            }
+            case 'error': {
+                const { detail, terminal } = providerError(message.error);
+                if (!providerReady || terminal) {
+                    stopped = true;
+                    ws?.close();
+                    close(`Voice provider error: ${detail}`);
+                }
+                else {
+                    state('connected', detail);
+                }
+                break;
+            }
+        }
+    }
+    function handleClientFrame(frame) {
+        if (tools.receive(frame))
+            return;
+        if (frame.type === 'realtime.control' && frame.action === 'stop') {
+            stopped = true;
+            ws?.close();
+            close('ended');
+            return;
+        }
+        if (frame.type === 'realtime.control' && frame.action === 'interrupt') {
+            const cancel = responseActive;
+            interrupt();
+            if (cancel && ws?.readyState === WebSocket.OPEN)
+                ws.send(JSON.stringify({ type: 'response.cancel' }));
+            return;
+        }
+        if (ws?.readyState !== WebSocket.OPEN)
+            return;
+        if (frame.type === 'realtime.audio') {
+            ws.send(JSON.stringify({ type: 'input_audio_buffer.append', audio: frame.data }));
+        }
+        else if (frame.type === 'realtime.say') {
+            ws.send(JSON.stringify({
+                type: 'conversation.item.create',
+                item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: frame.text }] },
+            }));
+            ws.send(JSON.stringify({ type: 'response.create' }));
+        }
+        // mute/unmute are enforced on the phone's capture side; nothing to forward.
+    }
+    function connectProvider(key) {
+        if (stopped)
+            return;
+        providerReady = false;
+        state('connecting');
+        const current = new WebSocket(PROVIDER_URL, {
+            headers: { Authorization: `Bearer ${key}` },
+            maxPayload: 4 * 1024 * 1024,
+        });
+        ws = current;
+        let downHandled = false;
+        const onDown = (reason) => {
+            if (downHandled || stopped || ws !== current)
+                return;
+            downHandled = true;
+            providerReady = false;
+            clearTimeout(stableTimer);
+            interrupt();
+            if (providerReconnects >= 2) {
+                close(reason);
+                return;
+            }
+            providerReconnects++;
+            state('connecting', 'Voice provider reconnecting');
+            reconnectTimer = setTimeout(() => connectProvider(key), providerReconnects * 500);
+        };
+        current.on('open', () => {
+            if (stopped || ws !== current)
+                return;
+            current.send(JSON.stringify({
+                type: 'session.update',
+                session: {
+                    type: 'realtime',
+                    model: MODEL,
+                    instructions: PROMPT + currentContext,
+                    output_modalities: ['audio'],
+                    audio: {
+                        input: {
+                            format: { type: 'audio/pcm', rate: RATE },
+                            transcription: { model: 'gpt-4o-mini-transcribe' },
+                            turn_detection: {
+                                type: 'server_vad',
+                                threshold: 0.9,
+                                silence_duration_ms: 700,
+                                prefix_padding_ms: 300,
+                                create_response: true,
+                                interrupt_response: true,
+                            },
+                        },
+                        output: { format: { type: 'audio/pcm' }, voice: options.voice ?? 'marin' },
+                    },
+                    tools: providerTools,
+                },
+            }));
+        });
+        current.on('message', (data) => {
+            if (!stopped && !downHandled && ws === current)
+                handleOpenAiEvent(String(data));
+        });
+        // ws suppresses 'error' and 'close' once this is handled, so the failure
+        // path is driven from here. Auth and permission refusals are terminal:
+        // retrying an out-of-credits account only delays the real message.
+        current.on('unexpected-response', (_request, response) => {
+            let body = '';
+            response.on('data', (chunk) => { if (Buffer.byteLength(body) + chunk.length > 16384) {
+                response.destroy();
+                current.terminate();
+                close('Voice provider refusal exceeded its limit.');
+                return;
+            } body += chunk; });
+            response.on('end', () => {
+                current.terminate();
+                if (stopped || ws !== current)
+                    return;
+                const reason = providerRefusal(response.statusCode, body);
+                if (response.statusCode === 401 || response.statusCode === 403)
+                    close(reason);
+                else
+                    onDown(reason);
+            });
+        });
+        current.on('close', (code, reasonBuffer) => {
+            if (stopped || ws !== current)
+                return;
+            const reason = cleanProviderProse(String(reasonBuffer), '', 160);
+            const detail = `OpenAI session ended (${code})${reason ? `: ${reason}` : '.'}`;
+            if (providerError(reason).terminal) {
+                stopped = true;
+                close(detail);
+            }
+            else {
+                onDown(detail);
+            }
+        });
+        current.on('error', (error) => {
+            if (!stopped && ws === current)
+                state('connecting', `Voice provider connection interrupted: ${cleanProviderProse(error.message, 'connection error', 160)}`);
+        });
+    }
+    connectProvider(options.key);
+    return { receive: handleClientFrame, close: (reason = 'ended') => { stopped = true; ws?.close(); close(reason); } };
+}

package/dist/providers/xai.d.ts

@@ -0,0 +1,6 @@
+import type { AdapterOptions } from '../adapter.ts';
+import type { RealtimeClientFrame } from '../frames.ts';
+export declare function createAdapter(options: AdapterOptions): {
+    receive(frame: RealtimeClientFrame): void;
+    close(reason?: string): void;
+};