discoclaw 1.2.3 → 1.3.0

package/dist/voice/providers/gemini-live-provider.js

@@ -0,0 +1,481 @@
+/**
+ * GeminiLiveProvider — bidirectional WebSocket session wrapper for the
+ * Gemini Multimodal Live API.
+ *
+ * Phase 1.1: standalone session management with a clean interface.
+ * Phase 1.2 (GeminiLiveResponder) and Phase 1.3 (pipeline integration)
+ * will consume this class.
+ *
+ * The provider manages:
+ * - WebSocket connection lifecycle (connect / disconnect)
+ * - Session setup (model, generation config, system instruction)
+ * - Sending audio input (PCM → base64-encoded chunks)
+ * - Receiving server events (audio output, text, turn completion, errors)
+ * - Reconnection with exponential backoff
+ *
+ * Audio format: Gemini Live expects 16 kHz mono PCM s16le input and
+ * returns 24 kHz mono PCM s16le output (configurable via responseModalities).
+ */
+import WebSocket from 'ws';
+import { DEFAULT_GEMINI_LIVE_MODEL, supportsGeminiLiveIncrementalClientContent, } from './gemini-live-types.js';
+import { GeminiLiveTokenEstimator } from './gemini-live-token-estimator.js';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const GEMINI_LIVE_WS_BASE = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
+const MAX_RETRIES = 3;
+const BASE_BACKOFF_MS = 500;
+/** Default session rotation threshold — 13 minutes (Gemini sessions cap at ~15 min). */
+const DEFAULT_SESSION_ROTATION_MS = 780_000;
+/** Session resumption handles remain valid for roughly 2 hours after termination. */
+const RESUME_HANDLE_TTL_MS = 2 * 60 * 60 * 1000;
+// ---------------------------------------------------------------------------
+// Provider
+// ---------------------------------------------------------------------------
+export class GeminiLiveProvider {
+    apiKey;
+    model;
+    log;
+    systemInstruction;
+    responseModalities;
+    voiceName;
+    tools;
+    wsFactory;
+    sessionRotationMs;
+    initialHistoryInClientContent;
+    ws = null;
+    _state = 'idle';
+    retryCount = 0;
+    resumeHandle = null;
+    resumeHandleUpdatedAt = 0;
+    listener = null;
+    sessionStartedAt = 0;
+    rotationTimer = null;
+    tokenEstimator;
+    /** Tool call IDs dispatched in the current session but not yet responded to. */
+    inflightToolCalls = new Set();
+    constructor(opts) {
+        this.apiKey = opts.apiKey;
+        this.model = opts.model ?? DEFAULT_GEMINI_LIVE_MODEL;
+        this.log = opts.log;
+        this.systemInstruction = opts.systemInstruction;
+        this.responseModalities = opts.responseModalities ?? ['AUDIO'];
+        this.voiceName = opts.voiceName;
+        this.tools = opts.tools;
+        this.wsFactory = opts.wsFactory ?? ((url) => new WebSocket(url));
+        this.sessionRotationMs = opts.sessionRotationMs ?? DEFAULT_SESSION_ROTATION_MS;
+        this.tokenEstimator = new GeminiLiveTokenEstimator(opts.tokenBudget);
+        this.initialHistoryInClientContent = opts.initialHistoryInClientContent ?? false;
+    }
+    /** Current connection state. */
+    get state() {
+        return this._state;
+    }
+    /** Register a listener for server events. Only one listener at a time. */
+    onEvent(callback) {
+        this.listener = callback;
+    }
+    /** Connect to the Gemini Live API and perform session setup. */
+    async connect() {
+        if (this._state === 'open' || this._state === 'connecting' || this._state === 'setup')
+            return;
+        this._state = 'connecting';
+        this.retryCount = 0;
+        await this.doConnect();
+    }
+    /**
+     * Send a chunk of PCM audio to the session.
+     * The buffer is base64-encoded and sent as a realtimeInput message.
+     */
+    sendAudio(pcm) {
+        if (this._state !== 'open') {
+            throw new Error('Cannot sendAudio before connect() completes or after disconnect()');
+        }
+        this.tokenEstimator.addInputAudio(pcm.length);
+        this.checkTokenThreshold();
+        this.ws.send(JSON.stringify({
+            realtimeInput: {
+                audio: {
+                    mimeType: 'audio/pcm;rate=16000',
+                    data: pcm.toString('base64'),
+                },
+            },
+        }));
+    }
+    /** Signal that the current realtime audio stream has ended so Gemini can flush buffered input. */
+    sendAudioStreamEnd() {
+        if (this._state !== 'open') {
+            throw new Error('Cannot sendAudioStreamEnd before connect() completes or after disconnect()');
+        }
+        this.ws.send(JSON.stringify({
+            realtimeInput: {
+                audioStreamEnd: true,
+            },
+        }));
+    }
+    /** Send a text message to the session. */
+    sendText(text) {
+        if (this._state !== 'open') {
+            throw new Error('Cannot sendText before connect() completes or after disconnect()');
+        }
+        this.tokenEstimator.addText(text);
+        this.checkTokenThreshold();
+        if (supportsGeminiLiveIncrementalClientContent(this.model)) {
+            this.ws.send(JSON.stringify({
+                clientContent: {
+                    turns: [{ role: 'user', parts: [{ text }] }],
+                    turnComplete: true,
+                },
+            }));
+            return;
+        }
+        this.ws.send(JSON.stringify({
+            realtimeInput: {
+                text,
+            },
+        }));
+    }
+    /**
+     * Seed the session with prior conversation turns before realtime audio starts.
+     * For Gemini 3.1, this is the supported path for initial history backfill.
+     */
+    sendInitialHistory(turns) {
+        if (this._state !== 'open') {
+            throw new Error('Cannot sendInitialHistory before connect() completes or after disconnect()');
+        }
+        if (turns.length === 0)
+            return;
+        for (const turn of turns) {
+            for (const part of turn.parts) {
+                this.tokenEstimator.addText(part.text);
+            }
+        }
+        this.checkTokenThreshold();
+        this.ws.send(JSON.stringify({
+            clientContent: {
+                turns,
+                turnComplete: false,
+            },
+        }));
+    }
+    /**
+     * Send tool execution results back to the session.
+     * Each response is matched to its original function call by `id`.
+     * Silently drops responses for IDs that are no longer in-flight
+     * (e.g. from a previous session after rotation).
+     */
+    sendToolResponse(responses) {
+        if (this._state !== 'open') {
+            throw new Error('Cannot sendToolResponse before connect() completes or after disconnect()');
+        }
+        // Filter to only in-flight calls — stale responses from a previous session are dropped
+        const valid = responses.filter((r) => {
+            if (this.inflightToolCalls.has(r.id)) {
+                this.inflightToolCalls.delete(r.id);
+                this.tokenEstimator.addToolResponse(r.output);
+                return true;
+            }
+            this.log.warn({ id: r.id }, 'Gemini Live: dropping stale tool response (not in-flight)');
+            return false;
+        });
+        if (valid.length === 0)
+            return;
+        this.checkTokenThreshold();
+        this.ws.send(JSON.stringify({
+            toolResponse: {
+                functionResponses: valid.map((r) => ({
+                    id: r.id,
+                    name: r.name,
+                    response: {
+                        result: r.output,
+                        ...(r.scheduling ? { scheduling: r.scheduling } : {}),
+                    },
+                })),
+            },
+        }));
+    }
+    /** Disconnect the session and release resources. */
+    async disconnect() {
+        if (this._state === 'stopped' || this._state === 'idle')
+            return;
+        this._state = 'stopped';
+        this.cancelRotationTimer();
+        this.inflightToolCalls.clear();
+        this.tokenEstimator.reset();
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.close(1000, 'client disconnect');
+        }
+        this.ws = null;
+    }
+    /** Number of tool calls currently in-flight (dispatched but not yet responded). */
+    get inflightToolCallCount() {
+        return this.inflightToolCalls.size;
+    }
+    // -----------------------------------------------------------------------
+    // Internal
+    // -----------------------------------------------------------------------
+    buildUrl() {
+        return `${GEMINI_LIVE_WS_BASE}?key=${encodeURIComponent(this.apiKey)}`;
+    }
+    buildSetupMessage() {
+        const generationConfig = {
+            responseModalities: this.responseModalities,
+        };
+        if (this.voiceName) {
+            generationConfig.speechConfig = {
+                voiceConfig: { prebuiltVoiceConfig: { voiceName: this.voiceName } },
+            };
+        }
+        const setup = {
+            model: `models/${this.model}`,
+            generationConfig,
+            contextWindowCompression: { slidingWindow: {} },
+            realtimeInputConfig: {
+                activityHandling: 'START_OF_ACTIVITY_INTERRUPTS',
+            },
+            inputAudioTranscription: {},
+            outputAudioTranscription: {},
+        };
+        if (this.systemInstruction) {
+            setup.systemInstruction = {
+                parts: [{ text: this.systemInstruction }],
+            };
+        }
+        if (this.tools) {
+            setup.tools = [this.tools];
+        }
+        if (this.initialHistoryInClientContent) {
+            setup.historyConfig = {
+                initialHistoryInClientContent: true,
+            };
+        }
+        if (this.resumeHandle) {
+            const age = Date.now() - this.resumeHandleUpdatedAt;
+            if (age < RESUME_HANDLE_TTL_MS) {
+                setup.sessionResumption = { handle: this.resumeHandle };
+            }
+            else {
+                this.log.warn({ ageMs: age, ttlMs: RESUME_HANDLE_TTL_MS }, 'Gemini Live resume handle expired — starting fresh session');
+                this.resumeHandle = null;
+            }
+        }
+        return { setup };
+    }
+    doConnect() {
+        return new Promise((resolve, reject) => {
+            const url = this.buildUrl();
+            const ws = this.wsFactory(url);
+            this.ws = ws;
+            ws.on('open', () => {
+                this._state = 'setup';
+                this.log.info({ model: this.model }, 'Gemini Live WebSocket connected, sending setup');
+                ws.send(JSON.stringify(this.buildSetupMessage()));
+            });
+            ws.on('message', (data) => {
+                this.handleMessage(data, resolve);
+            });
+            ws.on('error', (err) => {
+                this.log.error({ err: err.message }, 'Gemini Live WebSocket error');
+            });
+            ws.on('close', (code, reason) => {
+                if (this._state === 'stopped') {
+                    reject(new Error('Gemini Live WebSocket closed: disconnect() called'));
+                    return;
+                }
+                if (this._state === 'connecting' || this._state === 'setup') {
+                    reject(new Error(`Gemini Live WebSocket closed during connect: code=${code} reason=${reason.toString()}`));
+                    return;
+                }
+                this.handleUnexpectedClose();
+            });
+        });
+    }
+    handleMessage(data, onSetupComplete) {
+        try {
+            const parsed = JSON.parse(String(data));
+            // Setup complete acknowledgement
+            if (parsed.setupComplete != null) {
+                const wasReconnect = this.retryCount > 0;
+                const attempt = this.retryCount;
+                this._state = 'open';
+                this.retryCount = 0;
+                this.sessionStartedAt = Date.now();
+                this.tokenEstimator.reset();
+                this.inflightToolCalls.clear();
+                this.scheduleRotation();
+                this.log.info('Gemini Live session setup complete');
+                if (wasReconnect) {
+                    this.emit({ type: 'reconnected', attempt });
+                }
+                this.emit({ type: 'setup_complete' });
+                onSetupComplete?.();
+                return;
+            }
+            // Server content (audio, text, turn signals)
+            if (parsed.serverContent != null) {
+                const sc = parsed.serverContent;
+                // Interrupted signal
+                if (sc.interrupted === true) {
+                    this.emit({ type: 'interrupted' });
+                }
+                // Input audio transcription (server-side STT of user speech)
+                const inputTranscription = this.extractTranscriptionText(sc.inputTranscription);
+                if (inputTranscription) {
+                    this.emit({ type: 'input_transcript', text: inputTranscription });
+                }
+                // Output transcription mirrors audio-only model replies without requiring TEXT modality.
+                // Do not count these as text tokens; the audio output is already accounted separately.
+                const outputTranscription = this.extractTranscriptionText(sc.outputTranscription);
+                if (outputTranscription) {
+                    this.emit({ type: 'text', text: outputTranscription });
+                }
+                // Turn complete signal
+                if (sc.turnComplete === true) {
+                    this.emit({ type: 'turn_complete' });
+                }
+                // Model turn with parts
+                const modelTurn = sc.modelTurn;
+                if (modelTurn?.parts) {
+                    for (const part of modelTurn.parts) {
+                        if (part.inlineData != null) {
+                            const inline = part.inlineData;
+                            if (inline.data) {
+                                const buf = Buffer.from(inline.data, 'base64');
+                                this.tokenEstimator.addOutputAudio(buf.length);
+                                this.emit({ type: 'audio', data: buf });
+                            }
+                        }
+                        if (typeof part.text === 'string') {
+                            this.tokenEstimator.addText(part.text);
+                            this.emit({ type: 'text', text: part.text });
+                        }
+                    }
+                    this.checkTokenThreshold();
+                }
+                return;
+            }
+            // Tool call from server — model wants to invoke a function
+            if (parsed.toolCall != null) {
+                const tc = parsed.toolCall;
+                if (Array.isArray(tc.functionCalls) && tc.functionCalls.length > 0) {
+                    const calls = tc.functionCalls.map((fc) => ({
+                        id: String(fc.id ?? ''),
+                        name: String(fc.name ?? ''),
+                        args: fc.args ?? {},
+                    }));
+                    // Track in-flight IDs and estimate tokens
+                    for (const call of calls) {
+                        this.inflightToolCalls.add(call.id);
+                        this.tokenEstimator.addToolCall(call.name, call.args);
+                    }
+                    this.log.info({ count: calls.length, names: calls.map((c) => c.name).join(','), inflight: this.inflightToolCalls.size }, 'Gemini Live tool call received');
+                    this.emit({ type: 'tool_call', functionCalls: calls });
+                }
+                return;
+            }
+            // Error from server
+            if (parsed.error != null) {
+                const err = parsed.error;
+                const errMsg = err.message ?? `code ${err.code ?? 'unknown'}`;
+                this.log.error({ error: parsed.error }, 'Gemini Live server error');
+                this.emit({ type: 'error', error: errMsg });
+                return;
+            }
+            // Session resumption update — store handle for reconnection
+            if (parsed.sessionResumptionUpdate != null) {
+                const update = parsed.sessionResumptionUpdate;
+                if (update.newHandle) {
+                    this.resumeHandle = update.newHandle;
+                    this.resumeHandleUpdatedAt = Date.now();
+                    this.log.info('Gemini Live session resume handle updated');
+                }
+                return;
+            }
+            // Unknown message shape — log for debugging
+            this.log.warn({ keys: Object.keys(parsed).join(',') }, 'Gemini Live: unrecognized message');
+        }
+        catch (err) {
+            this.log.error({ err }, 'Failed to parse Gemini Live message');
+        }
+    }
+    handleUnexpectedClose() {
+        this.cancelRotationTimer();
+        this.inflightToolCalls.clear();
+        if (this.retryCount >= MAX_RETRIES) {
+            this.log.error({ retries: this.retryCount }, 'Gemini Live exhausted reconnect retries');
+            this._state = 'stopped';
+            this.emit({ type: 'reconnect_failed', attempts: this.retryCount });
+            this.emit({ type: 'fallback_recommended', reason: 'exhausted reconnect retries' });
+            this.emit({ type: 'error', error: 'exhausted reconnect retries' });
+            return;
+        }
+        this.retryCount++;
+        const delay = BASE_BACKOFF_MS * 2 ** (this.retryCount - 1);
+        this.log.warn({ attempt: this.retryCount, maxRetries: MAX_RETRIES, delayMs: delay }, 'Gemini Live reconnecting after unexpected close');
+        this.emit({
+            type: 'reconnecting',
+            attempt: this.retryCount,
+            maxRetries: MAX_RETRIES,
+            hasResumeHandle: this.resumeHandle != null && (Date.now() - this.resumeHandleUpdatedAt) < RESUME_HANDLE_TTL_MS,
+        });
+        setTimeout(() => {
+            if (this._state === 'stopped')
+                return;
+            this._state = 'connecting';
+            this.doConnect().catch((err) => {
+                this.log.error({ err }, 'Gemini Live reconnect failed');
+                this.handleUnexpectedClose();
+            });
+        }, delay);
+    }
+    scheduleRotation() {
+        this.cancelRotationTimer();
+        if (!this.sessionRotationMs)
+            return;
+        this.rotationTimer = setTimeout(() => {
+            this.rotationTimer = null;
+            this.initiateGracefulReconnect();
+        }, this.sessionRotationMs);
+    }
+    cancelRotationTimer() {
+        if (this.rotationTimer != null) {
+            clearTimeout(this.rotationTimer);
+            this.rotationTimer = null;
+        }
+    }
+    initiateGracefulReconnect() {
+        if (this._state !== 'open')
+            return;
+        const sessionAgeMs = Date.now() - this.sessionStartedAt;
+        this.log.info({ sessionAgeMs }, 'Gemini Live session rotation — initiating graceful reconnect');
+        this.emit({ type: 'session_rotating', sessionAgeMs });
+        // Close the WebSocket; handleUnexpectedClose will reconnect with the resume handle.
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.close(1000, 'session rotation');
+        }
+    }
+    /** Check token thresholds and emit warnings when newly crossed. */
+    checkTokenThreshold() {
+        const threshold = this.tokenEstimator.checkThreshold();
+        if (threshold) {
+            const est = this.tokenEstimator.estimate;
+            this.log.warn({ threshold, estimatedTokens: est.total, text: est.textTokens, audio: est.audioTokens, tool: est.toolTokens }, 'Gemini Live token threshold crossed');
+            this.emit({ type: 'token_warning', estimatedTokens: est.total, threshold });
+            // At the compress threshold, proactively trigger session rotation
+            // to prevent server-side sliding window from silently dropping context.
+            if (threshold === 'compress' && this._state === 'open') {
+                this.log.info({ estimatedTokens: est.total }, 'Gemini Live: compress threshold reached — initiating proactive rotation');
+                this.initiateGracefulReconnect();
+            }
+        }
+    }
+    emit(event) {
+        this.listener?.(event);
+    }
+    extractTranscriptionText(value) {
+        if (value == null || typeof value !== 'object')
+            return null;
+        const text = value.text;
+        return typeof text === 'string' && text !== '' ? text : null;
+    }
+}