discoclaw 1.2.4 → 2.0.0

package/dist/voice/providers/gemini-live-responder.js

@@ -0,0 +1,267 @@
+/**
+ * GeminiLiveResponder — Discord audio output bridge for the Gemini Live voice path.
+ *
+ * Phase 1.2: receives audio/text events from GeminiLiveProvider, upsamples
+ * audio to Discord's 48 kHz stereo format, manages an AudioPlayer + PassThrough
+ * stream pipeline, and handles barge-in via Gemini's interrupted/turn_complete events.
+ */
+import { PassThrough } from 'node:stream';
+import { createAudioPlayer, createAudioResource, AudioPlayerStatus, StreamType, } from '@discordjs/voice';
+import { upsampleToDiscord } from '../voice-responder.js';
+// Gemini Live returns 24 kHz mono PCM s16le
+const GEMINI_OUTPUT_RATE = 24_000;
+const GEMINI_OUTPUT_CHANNELS = 1;
+const TRANSCRIPT_FLUSH_DELAY_MS = 50;
+// ---------------------------------------------------------------------------
+// Responder
+// ---------------------------------------------------------------------------
+export class GeminiLiveResponder {
+    log;
+    connection;
+    provider;
+    onBotResponse;
+    onInputTranscript;
+    onToolCall;
+    onSessionTerminated;
+    onFallbackRecommended;
+    onTokenWarning;
+    playerFactory;
+    player = null;
+    stream = null;
+    transcript = '';
+    transcriptFlushTimer = null;
+    turnCompletePending = false;
+    started = false;
+    constructor(opts) {
+        this.log = opts.log;
+        this.connection = opts.connection;
+        this.provider = opts.provider;
+        this.onBotResponse = opts.onBotResponse;
+        this.onInputTranscript = opts.onInputTranscript;
+        this.onToolCall = opts.onToolCall;
+        this.onSessionTerminated = opts.onSessionTerminated;
+        this.onFallbackRecommended = opts.onFallbackRecommended;
+        this.onTokenWarning = opts.onTokenWarning;
+        this.playerFactory = opts.createPlayer ?? (() => createAudioPlayer());
+    }
+    /** Create the player, subscribe to the connection, and listen for provider events. */
+    start() {
+        if (this.started)
+            return;
+        this.started = true;
+        this.player = this.playerFactory();
+        const subscription = this.connection.subscribe(this.player);
+        this.log.info({ subscribed: !!subscription }, 'gemini-live-responder: player subscription result');
+        this.player.on('stateChange', (oldState, newState) => {
+            this.log.info({ from: oldState.status, to: newState.status }, 'gemini-live-responder: player state change');
+        });
+        this.player.on('error', (err) => {
+            this.log.error({ err }, 'gemini-live-responder: audio player error');
+        });
+        this.provider.onEvent((event) => this.handleEvent(event));
+    }
+    /** Whether the bot is audibly speaking (Playing or Buffering). */
+    get isPlaying() {
+        if (!this.player)
+            return false;
+        const status = this.player.state.status;
+        return status === AudioPlayerStatus.Playing || status === AudioPlayerStatus.Buffering;
+    }
+    /** Stop playback and tear down the stream. */
+    stop() {
+        this.destroyStream();
+        this.cancelTranscriptFlush();
+        this.player?.stop();
+        this.transcript = '';
+        this.turnCompletePending = false;
+    }
+    /** Stop and release all resources. */
+    destroy() {
+        this.stop();
+        this.started = false;
+        this.player = null;
+    }
+    // -----------------------------------------------------------------------
+    // Internal
+    // -----------------------------------------------------------------------
+    handleEvent(event) {
+        switch (event.type) {
+            case 'audio':
+                this.handleAudio(event.data);
+                break;
+            case 'text':
+                this.transcript += event.text;
+                if (this.turnCompletePending) {
+                    this.scheduleTranscriptFlush();
+                }
+                break;
+            case 'input_transcript':
+                this.handleInputTranscript(event.text);
+                break;
+            case 'interrupted':
+                this.handleInterrupted();
+                break;
+            case 'turn_complete':
+                this.handleTurnComplete();
+                break;
+            case 'tool_call':
+                this.handleToolCall(event.functionCalls);
+                break;
+            case 'session_rotating':
+                this.handleSessionRotating();
+                break;
+            case 'reconnecting':
+                this.handleReconnecting(event.attempt, event.maxRetries, event.hasResumeHandle);
+                break;
+            case 'reconnected':
+                this.log.info({ attempt: event.attempt }, 'gemini-live-responder: session reconnected');
+                break;
+            case 'reconnect_failed':
+                this.handleReconnectFailed(event.attempts);
+                break;
+            case 'token_warning':
+                this.handleTokenWarning(event.estimatedTokens, event.threshold);
+                break;
+            case 'fallback_recommended':
+                this.handleFallbackRecommended(event.reason);
+                break;
+            default:
+                // setup_complete, error — not handled here
+                break;
+        }
+    }
+    handleAudio(data) {
+        // Lazily create the stream + resource on the first audio chunk of a turn
+        if (!this.stream) {
+            this.stream = new PassThrough();
+            const resource = createAudioResource(this.stream, {
+                inputType: StreamType.Raw,
+            });
+            this.player.play(resource);
+            this.log.info({}, 'gemini-live-responder: streaming playback started');
+        }
+        const upsampled = upsampleToDiscord(data, GEMINI_OUTPUT_RATE, GEMINI_OUTPUT_CHANNELS);
+        this.stream.write(upsampled);
+    }
+    handleInterrupted() {
+        this.log.info({}, 'gemini-live-responder: interrupted — stopping playback');
+        this.destroyStream();
+        this.cancelTranscriptFlush();
+        this.player?.stop();
+        this.transcript = '';
+        this.turnCompletePending = false;
+    }
+    handleInputTranscript(text) {
+        if (this.onInputTranscript) {
+            try {
+                this.onInputTranscript(text);
+            }
+            catch (err) {
+                this.log.warn({ err }, 'gemini-live-responder: onInputTranscript callback error');
+            }
+        }
+    }
+    handleToolCall(calls) {
+        this.log.info({ count: calls.length, names: calls.map((c) => c.name).join(',') }, 'gemini-live-responder: tool call received');
+        if (this.onToolCall) {
+            try {
+                this.onToolCall(calls);
+            }
+            catch (err) {
+                this.log.warn({ err }, 'gemini-live-responder: onToolCall callback error');
+            }
+        }
+    }
+    handleTurnComplete() {
+        this.log.info({}, 'gemini-live-responder: turn complete');
+        // End the stream gracefully so remaining buffered audio plays out
+        if (this.stream) {
+            this.stream.end();
+            this.stream = null;
+        }
+        // Fire the transcript callback
+        this.turnCompletePending = true;
+        this.scheduleTranscriptFlush();
+    }
+    handleSessionRotating() {
+        this.log.info({}, 'gemini-live-responder: planned session rotation — pausing playback');
+        this.destroyStream();
+        this.cancelTranscriptFlush();
+        this.player?.stop();
+        this.transcript = '';
+        this.turnCompletePending = false;
+    }
+    handleReconnecting(attempt, maxRetries, hasResumeHandle) {
+        this.log.info({ attempt, maxRetries, hasResumeHandle }, 'gemini-live-responder: session reconnecting — pausing playback');
+        this.destroyStream();
+        this.cancelTranscriptFlush();
+        this.player?.stop();
+        this.transcript = '';
+        this.turnCompletePending = false;
+    }
+    handleReconnectFailed(attempts) {
+        this.log.error({ attempts }, 'gemini-live-responder: session terminally failed — all reconnect retries exhausted');
+        this.stop();
+        if (this.onSessionTerminated) {
+            try {
+                this.onSessionTerminated();
+            }
+            catch (err) {
+                this.log.warn({ err }, 'gemini-live-responder: onSessionTerminated callback error');
+            }
+        }
+    }
+    handleTokenWarning(estimatedTokens, threshold) {
+        this.log.warn({ estimatedTokens, threshold }, 'gemini-live-responder: token usage approaching context window limit');
+        if (this.onTokenWarning) {
+            try {
+                this.onTokenWarning(estimatedTokens, threshold);
+            }
+            catch (err) {
+                this.log.warn({ err }, 'gemini-live-responder: onTokenWarning callback error');
+            }
+        }
+    }
+    handleFallbackRecommended(reason) {
+        this.log.warn({ reason }, 'gemini-live-responder: fallback to pipeline voice mode recommended');
+        if (this.onFallbackRecommended) {
+            try {
+                this.onFallbackRecommended(reason);
+            }
+            catch (err) {
+                this.log.warn({ err }, 'gemini-live-responder: onFallbackRecommended callback error');
+            }
+        }
+    }
+    destroyStream() {
+        if (this.stream) {
+            this.stream.destroy();
+            this.stream = null;
+        }
+    }
+    scheduleTranscriptFlush() {
+        this.cancelTranscriptFlush();
+        this.transcriptFlushTimer = setTimeout(() => {
+            this.transcriptFlushTimer = null;
+            if (!this.turnCompletePending)
+                return;
+            const text = this.transcript;
+            this.transcript = '';
+            this.turnCompletePending = false;
+            if (!text)
+                return;
+            try {
+                this.onBotResponse?.(text);
+            }
+            catch (err) {
+                this.log.warn({ err }, 'gemini-live-responder: onBotResponse callback error');
+            }
+        }, TRANSCRIPT_FLUSH_DELAY_MS);
+    }
+    cancelTranscriptFlush() {
+        if (this.transcriptFlushTimer) {
+            clearTimeout(this.transcriptFlushTimer);
+            this.transcriptFlushTimer = null;
+        }
+    }
+}