@byokit/realtime 0.1.0

package/dist/providers/gemini.js

@@ -0,0 +1,429 @@
+import WebSocket from 'ws';
+import { responseFence } from "../policy.js";
+import { cleanProse, providerRefusal as refusal } from "../prose.js";
+// Lifted wire behavior; this module is loaded only by the kit child process.
+export function createAdapter(options) {
+    const MODEL = options.model ?? 'gemini-3.8-live';
+    const ENDPOINT = options.endpoint ?? 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
+    const INPUT_RATE = 16000;
+    const OUTPUT_RATE = 24000;
+    let endAfterResponse = false;
+    const providerTools = options.tools.map((tool) => ({ type: 'function', ...tool }));
+    const currentContext = '';
+    const isExplicitHangup = options.hangup ?? (() => false);
+    const cleanProviderProse = (value, fallback, max) => cleanProse(value, fallback, max, options.redact);
+    const providerRefusal = (status, body) => refusal(status, body, options.redact);
+    const tools = options.bridge;
+    const geminiSchema = (value) => {
+        if (Array.isArray(value))
+            return value.map(geminiSchema);
+        if (value === null || typeof value !== 'object')
+            return value;
+        return Object.fromEntries(Object.entries(value)
+            .filter(([key]) => key !== 'additionalProperties')
+            .map(([key, entry]) => [
+            key,
+            key === 'type' && typeof entry === 'string' ? entry.toUpperCase() : geminiSchema(entry),
+        ]));
+    };
+    const GEMINI_TOOLS = [{
+            functionDeclarations: providerTools.map(({ type: _type, ...tool }) => ({ ...tool, parameters: geminiSchema(tool.parameters) })),
+        }];
+    const OUTPUT_FRAME_MS = 20;
+    const AUDIO_CHUNK_SIZE = OUTPUT_RATE * 2 * OUTPUT_FRAME_MS / 1000 * 4 / 3;
+    const MAX_PENDING_OUTPUT_BYTES = 4 * 1024 * 1024;
+    const outputQueue = [];
+    let outputBytes = 0;
+    let outputBlocked = false;
+    let outputTimer;
+    let outputDeadline = 0;
+    let outputPausedByClient = false;
+    function syncProviderReadState() {
+        if (ws?.readyState !== WebSocket.OPEN)
+            return;
+        if (outputBlocked || outputQueue.length > 0 || outputPausedByClient)
+            ws.pause();
+        else
+            ws.resume();
+    }
+    function flushOutput() {
+        outputTimer = undefined;
+        if (outputBlocked || outputQueue.length === 0) {
+            if (outputQueue.length === 0)
+                outputDeadline = 0;
+            syncProviderReadState();
+            return;
+        }
+        const index = outputPausedByClient ? outputQueue.findIndex((item) => item.force) : 0;
+        if (index === -1) {
+            syncProviderReadState();
+            return;
+        }
+        const [item] = outputQueue.splice(index, 1);
+        outputBytes -= item.line.length;
+        outputBlocked = !process.stdout.write(item.line, item.done);
+        if (item.delayMs > 0) {
+            if (outputDeadline === 0)
+                outputDeadline = performance.now();
+            outputDeadline += item.delayMs;
+        }
+        const delayMs = item.delayMs > 0 ? Math.max(0, outputDeadline - performance.now()) : 0;
+        outputTimer = setTimeout(flushOutput, delayMs);
+        syncProviderReadState();
+    }
+    process.stdout.on('drain', () => {
+        outputBlocked = false;
+        outputDeadline = 0;
+        if (outputTimer === undefined)
+            flushOutput();
+    });
+    const emit = (frame, done, force = false) => {
+        const line = `${JSON.stringify(frame)}\n`;
+        if (!force && outputBytes + line.length > MAX_PENDING_OUTPUT_BYTES) {
+            stopped = true;
+            ws?.close();
+            outputQueue.length = 0;
+            outputBytes = 0;
+            outputDeadline = 0;
+            close('Voice output buffer overflowed.', true);
+            return false;
+        }
+        const delayMs = frame.type === 'realtime.audio'
+            ? Math.max(1, Math.round(frame.data.length / 4 * 3 / (OUTPUT_RATE * 2) * 1000))
+            : 0;
+        outputQueue.push({ line, delayMs, done, force, audio: frame.type === 'realtime.audio' });
+        outputBytes += line.length;
+        if (outputTimer === undefined)
+            outputTimer = setTimeout(flushOutput, 0);
+        syncProviderReadState();
+        return true;
+    };
+    function clearQueuedAudio() {
+        for (let index = outputQueue.length - 1; index >= 0; index -= 1) {
+            if (!outputQueue[index].audio)
+                continue;
+            outputBytes -= outputQueue[index].line.length;
+            outputQueue.splice(index, 1);
+        }
+        if (!outputQueue.some((item) => item.audio))
+            outputDeadline = 0;
+    }
+    // Gemini can deliver several seconds in one provider event. Twenty-millisecond
+    // frames keep native admission smooth while the paced queue backpressures the
+    // provider WebSocket whenever the phone or app is not draining.
+    function chunkAudio(audio) {
+        const chunks = [];
+        for (let offset = 0; offset < audio.length; offset += AUDIO_CHUNK_SIZE)
+            chunks.push(audio.slice(offset, offset + AUDIO_CHUNK_SIZE));
+        return chunks;
+    }
+    const emitAudio = (audio) => {
+        for (const data of chunkAudio(audio)) {
+            if (!emit({ type: 'realtime.audio', data }))
+                break;
+        }
+    };
+    const state = (value, detail) => tools.state(value, detail);
+    const PROMPT = options.instructions;
+    const fence = responseFence();
+    const interrupt = () => { fence.interrupt(); clearTimeout(finishTimer); clearQueuedAudio(); inputTranscript = ''; outputTranscript = ''; turnThinking = false; emit({ type: 'realtime.audio.clear' }, undefined, true); };
+    let closing = false;
+    const close = (reason, force = false) => {
+        if (closing)
+            return;
+        closing = true;
+        stopped = true;
+        fence.interrupt();
+        clearTimeout(reconnectTimer);
+        clearTimeout(stableTimer);
+        clearTimeout(finishTimer);
+        tools.close();
+        emit({ type: 'realtime.closed', reason }, () => process.exit(0), force);
+    };
+    let ws;
+    let stopped = false;
+    let providerReady = false;
+    let sessionHandle = '';
+    const cancelledToolCalls = new Set();
+    const activeToolCalls = new Map();
+    let providerReconnects = 0;
+    let reconnectTimer;
+    let stableTimer;
+    /** A provider link alive this long was healthy; forget its retries. */
+    const PROVIDER_STABLE_AFTER_MS = 30000;
+    const text = (value) => String(value ?? '').trim();
+    function providerError(error) {
+        const raw = typeof error === 'string' ? error : error?.message ?? error?.code ?? 'provider error';
+        const detail = cleanProviderProse(raw, 'provider error', 200);
+        return { detail, terminal: /api key|auth|credit|quota|billing|permission|forbidden|invalid json|invalid payload|unknown name|unsupported/i.test(detail) };
+    }
+    const runTool = (name, input, operationId, signal) => tools.run(name, input, operationId, signal);
+    let inputTranscript = '';
+    let outputTranscript = '';
+    let turnThinking = false;
+    let finishTimer;
+    function finishTurn() {
+        if (outputTranscript.trim())
+            tools.answered();
+        if (isExplicitHangup(inputTranscript))
+            endAfterResponse = true;
+        if (inputTranscript.trim())
+            emit({ type: 'realtime.transcript', role: 'user', text: inputTranscript.trim() });
+        if (outputTranscript.trim())
+            emit({ type: 'realtime.transcript', role: 'agent', text: outputTranscript.trim() });
+        inputTranscript = '';
+        outputTranscript = '';
+        turnThinking = false;
+        if (endAfterResponse) {
+            stopped = true;
+            ws?.close();
+            close('ended', true);
+        }
+        else {
+            state('connected');
+        }
+    }
+    function handleGeminiEvent(raw) {
+        let message;
+        try {
+            message = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        if (message.setupComplete) {
+            providerReady = true;
+            emit({ type: 'realtime.ready', inputRate: INPUT_RATE, outputRate: OUTPUT_RATE });
+            state('connected', providerReconnects === 0 ? undefined : 'Voice provider reconnected');
+            clearTimeout(stableTimer);
+            stableTimer = setTimeout(() => { providerReconnects = 0; }, PROVIDER_STABLE_AFTER_MS);
+        }
+        if (message.usageMetadata)
+            emit({ type: 'realtime.usage', usage: { seconds: 0, basis: 'tokens', inputTokens: message.usageMetadata.promptTokenCount ?? 0, outputTokens: message.usageMetadata.responseTokenCount ?? 0 } });
+        const content = message.serverContent;
+        if (content) {
+            if (content.interrupted === true) {
+                interrupt();
+            }
+            if (typeof content.inputTranscription?.text === 'string')
+                inputTranscript += content.inputTranscription.text;
+            if (fence.accepts() && typeof content.outputTranscription?.text === 'string')
+                outputTranscript += content.outputTranscription.text;
+            for (const part of fence.accepts() ? content.modelTurn?.parts ?? [] : []) {
+                const audio = part.inlineData?.data;
+                if (typeof audio === 'string' && audio) {
+                    if (!turnThinking) {
+                        state('thinking');
+                        turnThinking = true;
+                    }
+                    emitAudio(audio);
+                }
+            }
+            if (content.turnComplete === true) {
+                if (!fence.accepts()) {
+                    fence.begin();
+                    return;
+                }
+                // Transcription chunks are independent and have no final bit; give late chunks one short grace window.
+                clearTimeout(finishTimer);
+                finishTimer = setTimeout(finishTurn, 150);
+            }
+        }
+        if (typeof message.sessionResumptionUpdate?.newHandle === 'string' && message.sessionResumptionUpdate.resumable === true) {
+            sessionHandle = message.sessionResumptionUpdate.newHandle;
+        }
+        if (Array.isArray(message.toolCallCancellation?.ids)) {
+            // ponytail: completed Herdr side effects cannot be undone; cancellation aborts work still in flight.
+            for (const id of message.toolCallCancellation.ids) {
+                cancelledToolCalls.add(id);
+                activeToolCalls.get(id)?.abort();
+            }
+        }
+        if (fence.accepts() && Array.isArray(message.toolCall?.functionCalls)) {
+            const signal = fence.signal, current = ws;
+            void (async () => {
+                const functionResponses = (await Promise.all(message.toolCall.functionCalls.map(async (call) => {
+                    if (cancelledToolCalls.delete(call.id))
+                        return undefined;
+                    const controller = new AbortController();
+                    activeToolCalls.set(call.id, controller);
+                    let output;
+                    try {
+                        output = await runTool(call.name, call.args, call.id, AbortSignal.any([controller.signal, signal]));
+                    }
+                    catch (error) {
+                        if (cancelledToolCalls.delete(call.id) || error?.name === 'AbortError')
+                            return undefined;
+                        output = 'The request could not be completed. Please try again.';
+                    }
+                    finally {
+                        activeToolCalls.delete(call.id);
+                    }
+                    if (cancelledToolCalls.delete(call.id))
+                        return undefined;
+                    return { id: call.id, name: call.name, response: { result: text(output).slice(0, 24000) } };
+                }))).filter(Boolean);
+                if (stopped || signal.aborted || ws !== current || ws?.readyState !== WebSocket.OPEN || functionResponses.length === 0)
+                    return;
+                ws.send(JSON.stringify({ toolResponse: { functionResponses } }));
+            })();
+        }
+        if (message.goAway && ws?.readyState === WebSocket.OPEN) {
+            const current = ws;
+            const milliseconds = Math.max(0, (Number.parseFloat(message.goAway.timeLeft) || 1) * 1000 - 500);
+            setTimeout(() => {
+                if (!stopped && ws === current)
+                    current.close(1000, 'Session rotation');
+            }, milliseconds);
+        }
+        if (message.error) {
+            const { detail, terminal } = providerError(message.error);
+            if (!providerReady || terminal) {
+                stopped = true;
+                ws?.close();
+                close(`Voice provider error: ${detail}`, true);
+            }
+            else {
+                state('connected', detail);
+            }
+        }
+    }
+    function handleClientFrame(frame) {
+        if (tools.receive(frame))
+            return;
+        if (frame.type === 'realtime.control') {
+            if (frame.action === 'stop') {
+                stopped = true;
+                ws?.close();
+                outputQueue.length = 0;
+                outputBytes = 0;
+                outputDeadline = 0;
+                close('ended', true);
+            }
+            else if (frame.action === 'interrupt') {
+                interrupt();
+                sessionHandle = '';
+                // Automatic VAD owns activity signals; rotate rather than invent a cancel packet.
+                ws?.resume();
+                ws?.close(1000, 'Interrupted');
+            }
+            else if (frame.action === 'pause_output') {
+                outputPausedByClient = true;
+                outputDeadline = 0;
+                syncProviderReadState();
+            }
+            else if (frame.action === 'resume_output') {
+                outputPausedByClient = false;
+                if (outputTimer === undefined)
+                    outputTimer = setTimeout(flushOutput, 0);
+                syncProviderReadState();
+            }
+            return;
+        }
+        if (ws?.readyState !== WebSocket.OPEN)
+            return;
+        if (frame.type === 'realtime.audio') {
+            ws.send(JSON.stringify({ realtimeInput: { audio: { data: frame.data, mimeType: `audio/pcm;rate=${INPUT_RATE}` } } }));
+        }
+        else if (frame.type === 'realtime.say') {
+            ws.send(JSON.stringify({
+                clientContent: {
+                    turns: [{ role: 'user', parts: [{ text: frame.text }] }],
+                    turnComplete: true,
+                },
+            }));
+        }
+        // mute/unmute are enforced on the phone's capture side; nothing to forward.
+    }
+    function connectProvider(key) {
+        if (stopped)
+            return;
+        clearTimeout(reconnectTimer);
+        clearTimeout(stableTimer);
+        providerReady = false;
+        fence.begin();
+        state('connecting', providerReconnects === 0 ? undefined : 'Voice provider reconnecting');
+        const current = new WebSocket(`${ENDPOINT}?key=${encodeURIComponent(key)}`, { maxPayload: 4 * 1024 * 1024 });
+        ws = current;
+        let downHandled = false;
+        const onDown = (reason) => {
+            if (downHandled || stopped || ws !== current)
+                return;
+            downHandled = true;
+            providerReady = false;
+            clearTimeout(stableTimer);
+            interrupt();
+            state('connecting', 'Voice provider reconnecting');
+            if (providerReconnects >= 2) {
+                close(reason, true);
+                return;
+            }
+            providerReconnects += 1;
+            reconnectTimer = setTimeout(() => connectProvider(key), providerReconnects * 500);
+        };
+        current.on('open', () => {
+            if (stopped || ws !== current)
+                return;
+            current.send(JSON.stringify({
+                setup: {
+                    model: `models/${MODEL}`,
+                    generationConfig: {
+                        responseModalities: ['AUDIO'],
+                        speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: options.voice ?? 'Kore' } } },
+                    },
+                    systemInstruction: { parts: [{ text: PROMPT + currentContext }] },
+                    inputAudioTranscription: {},
+                    outputAudioTranscription: {},
+                    sessionResumption: sessionHandle ? { handle: sessionHandle } : {},
+                    contextWindowCompression: { slidingWindow: {} },
+                    tools: GEMINI_TOOLS,
+                },
+            }));
+        });
+        current.on('message', (data) => {
+            if (!stopped && !downHandled && ws === current)
+                handleGeminiEvent(String(data));
+        });
+        // ws suppresses 'error' and 'close' once this is handled, so the failure
+        // path is driven from here. Auth and permission refusals are terminal:
+        // retrying an out-of-credits account only delays the real message.
+        current.on('unexpected-response', (_request, response) => {
+            let body = '';
+            response.on('data', (chunk) => { if (Buffer.byteLength(body) + chunk.length > 16384) {
+                response.destroy();
+                current.terminate();
+                close('Voice provider refusal exceeded its limit.');
+                return;
+            } body += chunk; });
+            response.on('end', () => {
+                current.terminate();
+                if (stopped || ws !== current)
+                    return;
+                const reason = providerRefusal(response.statusCode, body);
+                if (response.statusCode === 401 || response.statusCode === 403)
+                    close(reason, true);
+                else
+                    onDown(reason);
+            });
+        });
+        current.on('close', (code, reasonBuffer) => {
+            if (stopped || ws !== current)
+                return;
+            const reason = cleanProviderProse(String(reasonBuffer), '', 160);
+            const detail = `The voice provider disconnected (${code})${reason ? `: ${reason}` : '.'}`;
+            if (providerError(reason).terminal) {
+                stopped = true;
+                close(detail, true);
+            }
+            else {
+                onDown(detail);
+            }
+        });
+        current.on('error', (error) => {
+            if (!stopped && ws === current)
+                state('connecting', `Voice provider connection interrupted: ${cleanProviderProse(error.message, 'connection error', 160)}`);
+        });
+    }
+    connectProvider(options.key);
+    return { receive: handleClientFrame, close: (reason = 'ended') => { stopped = true; ws?.close(); close(reason); } };
+}

package/dist/providers/openai.d.ts

@@ -0,0 +1,6 @@
+import type { AdapterOptions } from '../adapter.ts';
+import type { RealtimeClientFrame } from '../frames.ts';
+export declare function createAdapter(options: AdapterOptions): {
+    receive(frame: RealtimeClientFrame): void;
+    close(reason?: string): void;
+};