discoclaw 1.2.4 → 1.3.0

package/dist/voice/audio-pipeline.js

@@ -12,6 +12,11 @@ import { createSttProvider } from './stt-factory.js';
 import { createTtsProvider } from './tts-factory.js';
 import { VoiceResponder } from './voice-responder.js';
 import { ConversationBuffer } from './conversation-buffer.js';
+import { GeminiLiveProvider } from './providers/gemini-live-provider.js';
+import { GeminiLiveResponder } from './providers/gemini-live-responder.js';
+import { DEFAULT_GEMINI_LIVE_MODEL, normalizeGeminiLiveModel, supportsGeminiLiveAsyncFunctionCalling, } from './providers/gemini-live-types.js';
+import { buildGeminiToolDeclarations, buildToolSchemas, OPENAI_TO_DISCO_NAME } from '../runtime/openai-tool-schemas.js';
+import { executeToolCall } from '../runtime/openai-tool-exec.js';
 // ---------------------------------------------------------------------------
 // AudioPipelineManager
 // ---------------------------------------------------------------------------
@@ -31,6 +36,13 @@ export class AudioPipelineManager {
     transcriptMirror;
     botDisplayName;
     backfill;
+    buildGeminiSystemInstruction;
+    voiceProvider;
+    geminiApiKey;
+    enabledTools;
+    silentTools;
+    sessionRotationMs;
+    onFallbackTriggered;
     pipelines = new Map();
     /** Re-entrancy guard: VoiceConnection.subscribe() can synchronously fire stateChange→Ready. */
     starting = new Set();
@@ -50,6 +62,14 @@ export class AudioPipelineManager {
         this.transcriptMirror = opts.transcriptMirror;
         this.botDisplayName = opts.botDisplayName ?? 'Bot';
         this.backfill = opts.backfill;
+        this.buildGeminiSystemInstruction = opts.buildGeminiSystemInstruction;
+        this.voiceProvider = opts.voiceProvider ?? 'pipeline';
+        this.geminiApiKey = opts.geminiApiKey;
+        this.enabledTools = opts.enabledTools ?? [];
+        this.silentTools = new Set(opts.silentTools ?? []);
+        this.sessionRotationMs = opts.sessionRotationMs;
+        this.onFallbackTriggered = opts.onFallbackTriggered;
+        this.log.info({ voiceProvider: this.voiceProvider }, 'audio pipeline manager initialized');
     }
     /**
      * Attach to a VoiceConnection and auto-manage the audio pipeline
@@ -67,8 +87,8 @@ export class AudioPipelineManager {
             }
         });
     }
-    /** Start the audio receive pipeline for a guild. */
-    async startPipeline(guildId, connection) {
+    /** Start the audio receive pipeline for a guild. Pass `forceMode` to override the configured provider (used during fallback). */
+    async startPipeline(guildId, connection, forceMode) {
         // Re-entrancy guard: VoiceConnection.subscribe() (called when wiring the
         // AudioPlayer) synchronously fires a stateChange→Ready event, which would
         // re-invoke startPipeline and recurse infinitely.
@@ -80,7 +100,175 @@ export class AudioPipelineManager {
             this.log.info({ guildId }, 'stopping existing pipeline before restart');
             await this.stopPipeline(guildId);
         }
+        const effectiveMode = forceMode ?? this.voiceProvider;
         try {
+            // ----- gemini-live mode: skip STT/TTS, use GeminiLiveProvider directly -----
+            if (effectiveMode === 'gemini-live') {
+                const apiKey = this.geminiApiKey;
+                if (!apiKey)
+                    throw new Error('geminiApiKey is required for gemini-live voice provider');
+                const buffer = new ConversationBuffer();
+                if (this.backfill) {
+                    try {
+                        const turns = await this.backfill();
+                        buffer.backfill(turns);
+                        this.log.info({ guildId, turns: turns.length }, 'gemini-live conversation buffer backfilled');
+                    }
+                    catch (err) {
+                        this.log.warn({ guildId, err }, 'gemini-live conversation backfill failed — proceeding with empty history');
+                    }
+                }
+                const geminiLiveModel = normalizeGeminiLiveModel(this.runtimeModel) ?? DEFAULT_GEMINI_LIVE_MODEL;
+                const supportsAsyncFunctionCalling = supportsGeminiLiveAsyncFunctionCalling(geminiLiveModel);
+                const tools = buildGeminiToolDeclarations(this.enabledTools, { nonBlocking: supportsAsyncFunctionCalling });
+                const systemInstruction = await this.buildGeminiSystemInstruction?.();
+                const initialHistory = toGeminiLiveHistoryTurns(buffer.toTurns());
+                const provider = new GeminiLiveProvider({
+                    apiKey,
+                    log: this.log,
+                    model: geminiLiveModel,
+                    systemInstruction,
+                    responseModalities: ['AUDIO'],
+                    tools,
+                    initialHistoryInClientContent: initialHistory.length > 0,
+                    sessionRotationMs: this.sessionRotationMs,
+                });
+                await provider.connect();
+                if (initialHistory.length > 0) {
+                    provider.sendInitialHistory(initialHistory);
+                    this.log.info({ guildId, turns: initialHistory.length }, 'gemini-live conversation history seeded');
+                }
+                if (!supportsAsyncFunctionCalling && this.silentTools.size > 0) {
+                    this.log.info({ guildId, model: geminiLiveModel, count: this.silentTools.size }, 'gemini-live: current model does not support scheduled tool responses; silent tool scheduling disabled');
+                }
+                const mirror = this.transcriptMirror;
+                const botName = this.botDisplayName;
+                let latestInputTranscript;
+                const responder = new GeminiLiveResponder({
+                    log: this.log,
+                    connection,
+                    provider,
+                    onBotResponse: mirror
+                        ? (text) => {
+                            if (latestInputTranscript && text.trim()) {
+                                buffer.push(latestInputTranscript, text);
+                                latestInputTranscript = undefined;
+                            }
+                            mirror.postBotResponse(botName, text).catch((err) => {
+                                this.log.warn({ guildId, err }, 'transcript-mirror: failed to post bot response');
+                            });
+                        }
+                        : (text) => {
+                            if (latestInputTranscript && text.trim()) {
+                                buffer.push(latestInputTranscript, text);
+                                latestInputTranscript = undefined;
+                            }
+                        },
+                    onInputTranscript: mirror
+                        ? (text) => {
+                            if (text.trim())
+                                latestInputTranscript = text.trim();
+                            mirror.postUserTranscription('User', text).catch((err) => {
+                                this.log.warn({ guildId, err }, 'transcript-mirror: failed to post user transcription');
+                            });
+                        }
+                        : (text) => {
+                            if (text.trim())
+                                latestInputTranscript = text.trim();
+                        },
+                    onSessionTerminated: () => {
+                        this.log.error({ guildId }, 'gemini-live session terminally failed — no fallback (fallback disabled)');
+                    },
+                    onFallbackRecommended: (reason) => {
+                        this.log.warn({ guildId, reason }, 'gemini-live: fallback recommended but fallback is disabled');
+                    },
+                    onTokenWarning: (estimatedTokens, threshold) => {
+                        this.log.warn({ guildId, estimatedTokens, threshold }, 'gemini-live: token usage approaching context window limit');
+                    },
+                    onToolCall: tools
+                        ? (calls) => {
+                            this.log.info({ guildId, count: calls.length, names: calls.map((c) => c.name).join(',') }, 'gemini-live: tool call received — dispatching');
+                            const allowedRoots = this.runtimeCwd ? [this.runtimeCwd] : [];
+                            const allowedToolNames = new Set(buildToolSchemas(this.enabledTools).map((t) => t.function.name));
+                            const logFn = (msg) => this.log.info({ guildId }, msg);
+                            const execOpts = { enableHybridPipeline: false, allowedToolNames };
+                            // Gemini 3.1 Live only supports synchronous function calling.
+                            // Gemini 2.5 Live can opt into NON_BLOCKING declarations and scheduled responses.
+                            void (async () => {
+                                const results = await Promise.all(calls.map(async (call) => {
+                                    const scheduling = supportsAsyncFunctionCalling
+                                        ? (this.isSilentTool(call.name) ? 'SILENT' : 'INTERRUPT')
+                                        : undefined;
+                                    try {
+                                        const res = await executeToolCall(call.name, call.args, allowedRoots, logFn, execOpts);
+                                        return { id: call.id, name: call.name, output: res.result, scheduling };
+                                    }
+                                    catch (err) {
+                                        const msg = err instanceof Error ? err.message : String(err);
+                                        return { id: call.id, name: call.name, output: `Error: ${msg}`, scheduling };
+                                    }
+                                }));
+                                const silentCount = supportsAsyncFunctionCalling
+                                    ? results.filter((r) => r.scheduling === 'SILENT').length
+                                    : 0;
+                                if (silentCount > 0) {
+                                    this.log.info({ guildId, count: silentCount }, 'gemini-live: SILENT tool execution complete — results scheduled silently');
+                                }
+                                try {
+                                    provider.sendToolResponse(results);
+                                }
+                                catch (err) {
+                                    this.log.warn({ guildId, err }, 'gemini-live: sendToolResponse failed (provider likely disconnected)');
+                                }
+                            })();
+                        }
+                        : undefined,
+                });
+                responder.start();
+                // SttProvider shim: bridges AudioReceiver frames to GeminiLiveProvider.sendAudio
+                const sttShim = {
+                    start: async () => { },
+                    stop: async () => { },
+                    onTranscription: () => { },
+                    feedAudio: (frame) => {
+                        try {
+                            provider.sendAudio(frame.buffer);
+                        }
+                        catch (err) {
+                            this.log.warn({ guildId, err }, 'gemini-live: sendAudio error (non-fatal)');
+                        }
+                    },
+                };
+                const receiver = new AudioReceiver({
+                    connection,
+                    allowedUserIds: this.allowedUserIds,
+                    sttProvider: sttShim,
+                    log: this.log,
+                    createDecoder: this.createDecoder,
+                    onUserSpeaking: () => { },
+                    onUserSilence: () => {
+                        try {
+                            provider.sendAudioStreamEnd();
+                        }
+                        catch (err) {
+                            this.log.warn({ guildId, err }, 'gemini-live: sendAudioStreamEnd error (non-fatal)');
+                        }
+                    },
+                });
+                receiver.start();
+                this.pipelines.set(guildId, {
+                    connection,
+                    sttProvider: sttShim,
+                    receiver,
+                    buffer,
+                    geminiProvider: provider,
+                    geminiResponder: responder,
+                    mode: 'gemini-live',
+                });
+                this.log.info({ guildId }, 'audio pipeline started (gemini-live)');
+                return;
+            }
+            // ----- default pipeline mode: STT/TTS/VoiceResponder -----
             const sttProvider = this.createStt(this.voiceConfig, this.log);
             const mirror = this.transcriptMirror;
             // Create conversation buffer and backfill history if available
@@ -166,11 +354,15 @@ export class AudioPipelineManager {
                 },
             });
             receiver.start();
-            this.pipelines.set(guildId, { connection, sttProvider, receiver, responder, buffer });
-            this.log.info({ guildId }, 'audio pipeline started');
+            this.pipelines.set(guildId, { connection, sttProvider, receiver, responder, buffer, mode: 'pipeline' });
+            this.log.info({ guildId, mode: effectiveMode }, 'audio pipeline started');
         }
         catch (err) {
             this.log.error({ guildId, err }, 'failed to start audio pipeline');
+            // Fallback disabled — gemini-live must succeed or the pipeline stays down
+            if (effectiveMode === 'gemini-live') {
+                this.log.error({ guildId }, 'gemini-live: connection failed — no fallback (fallback disabled)');
+            }
         }
         finally {
             this.starting.delete(guildId);
@@ -182,6 +374,10 @@ export class AudioPipelineManager {
         if (!pipeline)
             return;
         this.pipelines.delete(guildId);
+        pipeline.geminiResponder?.destroy();
+        if (pipeline.geminiProvider) {
+            await pipeline.geminiProvider.disconnect();
+        }
         pipeline.responder?.destroy();
         pipeline.receiver.stop();
         try {
@@ -205,16 +401,52 @@ export class AudioPipelineManager {
     get activePipelineCount() {
         return this.pipelines.size;
     }
+    /** Configured voice provider mode ('pipeline' or 'gemini-live'). */
+    get activeVoiceProvider() {
+        return this.voiceProvider;
+    }
+    /** Active mode for a specific guild (may differ from configured mode during fallback). */
+    pipelineMode(guildId) {
+        return this.pipelines.get(guildId)?.mode;
+    }
     /** Current Deepgram TTS voice model name. */
     get ttsVoice() {
         return this.voiceConfig.deepgramTtsVoice;
     }
+    /**
+     * Fall back from gemini-live to the standard pipeline for a guild.
+     * Stops the current gemini-live session and starts a standard STT/AI/TTS pipeline.
+     * No-op if no pipeline exists or the guild is already in standard mode.
+     */
+    async fallbackToPipeline(guildId, connection) {
+        const pipeline = this.pipelines.get(guildId);
+        if (!pipeline || pipeline.mode !== 'gemini-live')
+            return;
+        this.log.warn({ guildId }, 'gemini-live: initiating fallback to standard pipeline');
+        await this.stopPipeline(guildId);
+        await this.startPipeline(guildId, connection, 'pipeline');
+        if (this.hasPipeline(guildId)) {
+            this.log.info({ guildId }, 'gemini-live: fallback to standard pipeline succeeded');
+            this.onFallbackTriggered?.(guildId, 'pipeline');
+        }
+        else {
+            this.log.error({ guildId }, 'gemini-live: fallback to standard pipeline also failed — guild has no active pipeline');
+        }
+    }
+    isSilentTool(toolName) {
+        return this.silentTools.has(toolName) || this.silentTools.has(OPENAI_TO_DISCO_NAME[toolName] ?? toolName);
+    }
     /**
      * Update the Deepgram TTS voice and restart all active pipelines so the
-     * new voice takes effect immediately.
-     * @returns The number of pipelines that were restarted.
+     * new voice takes effect immediately. No-op in gemini-live mode (TTS is
+     * handled server-side).
+     * @returns The number of pipelines that were restarted (0 in gemini-live mode).
      */
     async setTtsVoice(voice) {
+        if (this.voiceProvider === 'gemini-live') {
+            this.log.info({ voice }, 'TTS voice change ignored — gemini-live mode uses server-side TTS');
+            return 0;
+        }
         this.voiceConfig = { ...this.voiceConfig, deepgramTtsVoice: voice };
         this.log.info({ voice }, 'TTS voice updated — restarting active pipelines');
         const entries = [...this.pipelines.entries()];
@@ -222,3 +454,11 @@ export class AudioPipelineManager {
         return entries.length;
     }
 }
+function toGeminiLiveHistoryTurns(turns) {
+    const history = [];
+    for (const turn of turns) {
+        history.push({ role: 'user', parts: [{ text: turn.user }] });
+        history.push({ role: 'model', parts: [{ text: turn.assistant }] });
+    }
+    return history;
+}