discoclaw 1.2.4 → 2.0.0

package/dist/tasks/task-action-read-ops.js

@@ -1,5 +1,6 @@
 import { runTaskSync } from './task-sync.js';
 import { reloadTagMapInPlace } from './tag-map.js';
+import { getTaskActionThreadMetadata } from './task-action-runner-types.js';
 function resolveTaskId(action) {
     return (action.taskId ?? '').trim();
 }
@@ -18,11 +19,16 @@ export async function handleTaskShow(action, _ctx, taskCtx) {
     ];
     if (task.owner)
         lines.push(`Owner: ${task.owner}`);
+    const thread = getTaskActionThreadMetadata(task);
+    if (thread)
+        lines.push(`External ref: ${thread.externalRef}`);
+    if (thread?.threadUrl)
+        lines.push(`Thread: ${thread.threadUrl}`);
     if (task.labels?.length)
         lines.push(`Labels: ${task.labels.join(', ')}`);
     if (task.description)
         lines.push(`\n${task.description.slice(0, 500)}`);
-    return { ok: true, summary: lines.join('\n') };
+    return { ok: true, summary: lines.join('\n'), ...(thread ? { thread } : {}) };
 }
 export async function handleTaskList(action, _ctx, taskCtx) {
     const tasks = taskCtx.store.list({

package/dist/tasks/task-action-runner-types.js

@@ -1 +1,19 @@
-export {};
+import { getThreadIdFromTask } from './thread-helpers.js';
+export function buildDiscordThreadUrl(guildId, threadId) {
+    return `https://discord.com/channels/${guildId}/${threadId}`;
+}
+export function getTaskActionThreadMetadata(task) {
+    const externalRef = task.external_ref?.trim() ?? '';
+    if (!externalRef)
+        return undefined;
+    const threadId = getThreadIdFromTask(task) ?? undefined;
+    const threadGuildId = task.thread_origin_guild?.trim() || undefined;
+    return {
+        externalRef,
+        ...(threadId ? { threadId } : {}),
+        ...(threadGuildId ? { threadGuildId } : {}),
+        ...(threadId && threadGuildId
+            ? { threadUrl: buildDiscordThreadUrl(threadGuildId, threadId) }
+            : {}),
+    };
+}

package/dist/voice/audio-pipeline.js

@@ -8,48 +8,55 @@
  */
 import { VoiceConnectionStatus } from '@discordjs/voice';
 import { AudioReceiver } from './audio-receiver.js';
-import { createSttProvider } from './stt-factory.js';
-import { createTtsProvider } from './tts-factory.js';
-import { VoiceResponder } from './voice-responder.js';
 import { ConversationBuffer } from './conversation-buffer.js';
+import { GeminiLiveProvider } from './providers/gemini-live-provider.js';
+import { GeminiLiveResponder } from './providers/gemini-live-responder.js';
+import { DEFAULT_GEMINI_LIVE_MODEL, normalizeGeminiLiveModel, supportsGeminiLiveAsyncFunctionCalling, } from './providers/gemini-live-types.js';
+import { buildGeminiToolDeclarations, buildToolSchemas, OPENAI_TO_DISCO_NAME } from '../runtime/openai-tool-schemas.js';
+import { executeToolCall } from '../runtime/openai-tool-exec.js';
 // ---------------------------------------------------------------------------
 // AudioPipelineManager
 // ---------------------------------------------------------------------------
 export class AudioPipelineManager {
     log;
-    voiceConfig;
     allowedUserIds;
     createDecoder;
     onTranscription;
-    createStt;
     invokeAi;
     runtime;
     runtimeModel;
     runtimeCwd;
     runtimeTimeoutMs;
-    createTts;
     transcriptMirror;
     botDisplayName;
     backfill;
+    buildGeminiSystemInstruction;
+    geminiApiKey;
+    enabledTools;
+    silentTools;
+    sessionRotationMs;
     pipelines = new Map();
     /** Re-entrancy guard: VoiceConnection.subscribe() can synchronously fire stateChange→Ready. */
     starting = new Set();
     constructor(opts) {
         this.log = opts.log;
-        this.voiceConfig = opts.voiceConfig;
         this.allowedUserIds = opts.allowedUserIds;
         this.createDecoder = opts.createDecoder;
         this.onTranscription = opts.onTranscription;
-        this.createStt = opts.createStt ?? createSttProvider;
         this.invokeAi = opts.invokeAi;
         this.runtime = opts.runtime;
         this.runtimeModel = opts.runtimeModel;
         this.runtimeCwd = opts.runtimeCwd;
         this.runtimeTimeoutMs = opts.runtimeTimeoutMs;
-        this.createTts = opts.createTts ?? createTtsProvider;
         this.transcriptMirror = opts.transcriptMirror;
         this.botDisplayName = opts.botDisplayName ?? 'Bot';
         this.backfill = opts.backfill;
+        this.buildGeminiSystemInstruction = opts.buildGeminiSystemInstruction;
+        this.geminiApiKey = opts.geminiApiKey;
+        this.enabledTools = opts.enabledTools ?? [];
+        this.silentTools = new Set(opts.silentTools ?? []);
+        this.sessionRotationMs = opts.sessionRotationMs;
+        this.log.info({ voiceProvider: 'gemini-live' }, 'audio pipeline manager initialized');
     }
     /**
      * Attach to a VoiceConnection and auto-manage the audio pipeline
@@ -67,7 +74,7 @@ export class AudioPipelineManager {
             }
         });
     }
-    /** Start the audio receive pipeline for a guild. */
+    /** Start the Gemini Live voice pipeline for a guild. */
     async startPipeline(guildId, connection) {
         // Re-entrancy guard: VoiceConnection.subscribe() (called when wiring the
         // AudioPlayer) synchronously fires a stateChange→Ready event, which would
@@ -81,96 +88,170 @@ export class AudioPipelineManager {
             await this.stopPipeline(guildId);
         }
         try {
-            const sttProvider = this.createStt(this.voiceConfig, this.log);
-            const mirror = this.transcriptMirror;
-            // Create conversation buffer and backfill history if available
-            let buffer;
-            if (this.invokeAi) {
-                buffer = new ConversationBuffer();
-                if (this.backfill) {
-                    try {
-                        const turns = await this.backfill();
-                        buffer.backfill(turns);
-                        this.log.info({ guildId, turns: turns.length }, 'conversation buffer backfilled');
-                    }
-                    catch (err) {
-                        this.log.warn({ guildId, err }, 'conversation backfill failed — proceeding with empty buffer');
-                    }
-                }
-            }
-            // Create VoiceResponder for the full conversation loop if invokeAi is configured
-            let responder;
-            if (this.invokeAi) {
+            const apiKey = this.geminiApiKey;
+            if (!apiKey)
+                throw new Error('geminiApiKey is required for gemini-live voice provider');
+            const buffer = new ConversationBuffer();
+            if (this.backfill) {
                 try {
-                    const tts = this.createTts(this.voiceConfig, this.log);
-                    const botName = this.botDisplayName;
-                    responder = new VoiceResponder({
-                        log: this.log,
-                        tts,
-                        connection,
-                        invokeAi: this.invokeAi,
-                        onBotResponse: mirror
-                            ? (text) => {
-                                mirror.postBotResponse(botName, text).catch((err) => {
-                                    this.log.warn({ guildId, err }, 'transcript-mirror: failed to post bot response');
-                                });
-                            }
-                            : undefined,
-                        buffer,
-                    });
-                    this.log.info({ guildId }, 'voice responder created');
+                    const turns = await this.backfill();
+                    buffer.backfill(turns);
+                    this.log.info({ guildId, turns: turns.length }, 'gemini-live conversation buffer backfilled');
                 }
                 catch (err) {
-                    this.log.warn({ guildId, err }, 'failed to create voice responder — continuing with STT-only mode');
+                    this.log.warn({ guildId, err }, 'gemini-live conversation backfill failed — proceeding with empty history');
                 }
             }
-            // Wire transcription callback — fires the external callback, transcript mirror, and responder
-            const onTranscriptionCb = this.onTranscription;
-            if (onTranscriptionCb || responder || mirror) {
-                sttProvider.onTranscription((result) => {
-                    if (onTranscriptionCb) {
-                        onTranscriptionCb(guildId, result);
-                    }
-                    // STT-confirmed barge-in: any transcription (interim or final) with
-                    // non-empty text stops ongoing playback. Echo produces empty
-                    // transcriptions; real speech produces non-empty ones.
-                    if (result.text.trim() && responder?.isPlaying) {
-                        this.log.info({ guildId }, 'barge-in detected');
-                        responder.stop();
-                    }
-                    if (result.isFinal && result.text.trim()) {
-                        if (mirror) {
-                            mirror.postUserTranscription('User', result.text).catch((err) => {
-                                this.log.warn({ guildId, err }, 'transcript-mirror: failed to post user transcription');
-                            });
+            const geminiLiveModel = normalizeGeminiLiveModel(this.runtimeModel) ?? DEFAULT_GEMINI_LIVE_MODEL;
+            const supportsAsyncFunctionCalling = supportsGeminiLiveAsyncFunctionCalling(geminiLiveModel);
+            const tools = buildGeminiToolDeclarations(this.enabledTools, { nonBlocking: supportsAsyncFunctionCalling });
+            const systemInstruction = await this.buildGeminiSystemInstruction?.();
+            const initialHistory = toGeminiLiveHistoryTurns(buffer.toTurns());
+            const provider = new GeminiLiveProvider({
+                apiKey,
+                log: this.log,
+                model: geminiLiveModel,
+                systemInstruction,
+                responseModalities: ['AUDIO'],
+                tools,
+                initialHistoryInClientContent: initialHistory.length > 0,
+                sessionRotationMs: this.sessionRotationMs,
+            });
+            await provider.connect();
+            if (initialHistory.length > 0) {
+                provider.sendInitialHistory(initialHistory);
+                this.log.info({ guildId, turns: initialHistory.length }, 'gemini-live conversation history seeded');
+            }
+            if (!supportsAsyncFunctionCalling && this.silentTools.size > 0) {
+                this.log.info({ guildId, model: geminiLiveModel, count: this.silentTools.size }, 'gemini-live: current model does not support scheduled tool responses; silent tool scheduling disabled');
+            }
+            const mirror = this.transcriptMirror;
+            const botName = this.botDisplayName;
+            let latestInputTranscript;
+            const responder = new GeminiLiveResponder({
+                log: this.log,
+                connection,
+                provider,
+                onBotResponse: mirror
+                    ? (text) => {
+                        if (latestInputTranscript && text.trim()) {
+                            buffer.push(latestInputTranscript, text);
+                            latestInputTranscript = undefined;
                         }
-                        if (responder) {
-                            responder.handleTranscription(result.text).catch((err) => {
-                                this.log.error({ guildId, err }, 'voice-responder: handleTranscription failed');
-                            });
+                        mirror.postBotResponse(botName, text).catch((err) => {
+                            this.log.warn({ guildId, err }, 'transcript-mirror: failed to post bot response');
+                        });
+                    }
+                    : (text) => {
+                        if (latestInputTranscript && text.trim()) {
+                            buffer.push(latestInputTranscript, text);
+                            latestInputTranscript = undefined;
                         }
+                    },
+                onInputTranscript: mirror
+                    ? (text) => {
+                        if (text.trim())
+                            latestInputTranscript = text.trim();
+                        mirror.postUserTranscription('User', text).catch((err) => {
+                            this.log.warn({ guildId, err }, 'transcript-mirror: failed to post user transcription');
+                        });
                     }
-                });
-            }
-            await sttProvider.start();
+                    : (text) => {
+                        if (text.trim())
+                            latestInputTranscript = text.trim();
+                    },
+                onSessionTerminated: () => {
+                    this.log.error({ guildId }, 'gemini-live session terminally failed — no fallback');
+                },
+                onFallbackRecommended: (reason) => {
+                    this.log.warn({ guildId, reason }, 'gemini-live: fallback recommended but the legacy pipeline has been removed');
+                },
+                onTokenWarning: (estimatedTokens, threshold) => {
+                    this.log.warn({ guildId, estimatedTokens, threshold }, 'gemini-live: token usage approaching context window limit');
+                },
+                onToolCall: tools
+                    ? (calls) => {
+                        this.log.info({ guildId, count: calls.length, names: calls.map((c) => c.name).join(',') }, 'gemini-live: tool call received — dispatching');
+                        const allowedRoots = this.runtimeCwd ? [this.runtimeCwd] : [];
+                        const allowedToolNames = new Set(buildToolSchemas(this.enabledTools).map((t) => t.function.name));
+                        const logFn = (msg) => this.log.info({ guildId }, msg);
+                        const execOpts = { enableHybridPipeline: false, allowedToolNames };
+                        void (async () => {
+                            const results = await Promise.all(calls.map(async (call) => {
+                                const scheduling = supportsAsyncFunctionCalling
+                                    ? (this.isSilentTool(call.name) ? 'SILENT' : 'INTERRUPT')
+                                    : undefined;
+                                try {
+                                    const res = await executeToolCall(call.name, call.args, allowedRoots, logFn, execOpts);
+                                    return { id: call.id, name: call.name, output: res.result, scheduling };
+                                }
+                                catch (err) {
+                                    const msg = err instanceof Error ? err.message : String(err);
+                                    return { id: call.id, name: call.name, output: `Error: ${msg}`, scheduling };
+                                }
+                            }));
+                            const silentCount = supportsAsyncFunctionCalling
+                                ? results.filter((r) => r.scheduling === 'SILENT').length
+                                : 0;
+                            if (silentCount > 0) {
+                                this.log.info({ guildId, count: silentCount }, 'gemini-live: SILENT tool execution complete — results scheduled silently');
+                            }
+                            try {
+                                provider.sendToolResponse(results);
+                            }
+                            catch (err) {
+                                this.log.warn({ guildId, err }, 'gemini-live: sendToolResponse failed (provider likely disconnected)');
+                            }
+                        })();
+                    }
+                    : undefined,
+            });
+            responder.start();
+            // SttProvider shim: bridges AudioReceiver frames to GeminiLiveProvider.sendAudio
+            const sttShim = {
+                start: async () => { },
+                stop: async () => { },
+                onTranscription: () => { },
+                feedAudio: (frame) => {
+                    try {
+                        provider.sendAudio(frame.buffer);
+                    }
+                    catch (err) {
+                        this.log.warn({ guildId, err }, 'gemini-live: sendAudio error (non-fatal)');
+                    }
+                },
+            };
             const receiver = new AudioReceiver({
                 connection,
                 allowedUserIds: this.allowedUserIds,
-                sttProvider,
+                sttProvider: sttShim,
                 log: this.log,
                 createDecoder: this.createDecoder,
-                onUserSpeaking: (_userId) => {
-                    // Barge-in is now gated on STT transcription (see onTranscription
-                    // callback above). This callback is kept for AudioReceiver
-                    // subscription management.
+                onUserSpeaking: () => { },
+                onUserSilence: () => {
+                    try {
+                        provider.sendAudioStreamEnd();
+                    }
+                    catch (err) {
+                        this.log.warn({ guildId, err }, 'gemini-live: sendAudioStreamEnd error (non-fatal)');
+                    }
                 },
             });
             receiver.start();
-            this.pipelines.set(guildId, { connection, sttProvider, receiver, responder, buffer });
-            this.log.info({ guildId }, 'audio pipeline started');
+            this.pipelines.set(guildId, {
+                connection,
+                sttProvider: sttShim,
+                receiver,
+                buffer,
+                geminiProvider: provider,
+                geminiResponder: responder,
+                mode: 'gemini-live',
+            });
+            this.log.info({ guildId }, 'audio pipeline started (gemini-live)');
         }
         catch (err) {
             this.log.error({ guildId, err }, 'failed to start audio pipeline');
+            this.log.error({ guildId }, 'gemini-live: connection failed — no fallback available');
         }
         finally {
             this.starting.delete(guildId);
@@ -182,7 +263,10 @@ export class AudioPipelineManager {
         if (!pipeline)
             return;
         this.pipelines.delete(guildId);
-        pipeline.responder?.destroy();
+        pipeline.geminiResponder?.destroy();
+        if (pipeline.geminiProvider) {
+            await pipeline.geminiProvider.disconnect();
+        }
         pipeline.receiver.stop();
         try {
             await pipeline.sttProvider.stop();
@@ -205,20 +289,23 @@ export class AudioPipelineManager {
     get activePipelineCount() {
         return this.pipelines.size;
     }
-    /** Current Deepgram TTS voice model name. */
-    get ttsVoice() {
-        return this.voiceConfig.deepgramTtsVoice;
+    /** Configured voice provider mode. */
+    get activeVoiceProvider() {
+        return 'gemini-live';
     }
-    /**
-     * Update the Deepgram TTS voice and restart all active pipelines so the
-     * new voice takes effect immediately.
-     * @returns The number of pipelines that were restarted.
-     */
-    async setTtsVoice(voice) {
-        this.voiceConfig = { ...this.voiceConfig, deepgramTtsVoice: voice };
-        this.log.info({ voice }, 'TTS voice updated — restarting active pipelines');
-        const entries = [...this.pipelines.entries()];
-        await Promise.all(entries.map(([guildId, pipeline]) => this.startPipeline(guildId, pipeline.connection)));
-        return entries.length;
+    /** Active mode for a specific guild. */
+    pipelineMode(guildId) {
+        return this.pipelines.get(guildId)?.mode;
+    }
+    isSilentTool(toolName) {
+        return this.silentTools.has(toolName) || this.silentTools.has(OPENAI_TO_DISCO_NAME[toolName] ?? toolName);
+    }
+}
+function toGeminiLiveHistoryTurns(turns) {
+    const history = [];
+    for (const turn of turns) {
+        history.push({ role: 'user', parts: [{ text: turn.user }] });
+        history.push({ role: 'model', parts: [{ text: turn.assistant }] });
     }
+    return history;
 }

package/dist/voice/audio-receiver.js

@@ -22,6 +22,7 @@ export class AudioReceiver {
     log;
     createDecoder;
     onUserSpeaking;
+    onUserSilence;
     decoders = new Map();
     running = false;
     constructor(opts) {
@@ -31,6 +32,7 @@ export class AudioReceiver {
         this.log = opts.log;
         this.createDecoder = opts.createDecoder;
         this.onUserSpeaking = opts.onUserSpeaking;
+        this.onUserSilence = opts.onUserSilence;
     }
     /** Begin listening for audio from allowlisted users. */
     start() {
@@ -122,6 +124,12 @@ export class AudioReceiver {
             decoder.destroy();
             this.decoders.delete(userId);
             this.log.info({ userId }, 'cleaned up user audio decoder');
+            try {
+                this.onUserSilence?.(userId);
+            }
+            catch (err) {
+                this.log.error({ err, userId }, 'onUserSilence callback error');
+            }
         }
     }
 }

package/dist/voice/audio-receiver.test.js

@@ -224,6 +224,22 @@ describe('AudioReceiver', () => {
         expect(decoder.destroy).toHaveBeenCalled();
         expect(log.info).toHaveBeenCalledWith({ userId: '111' }, 'cleaned up user audio decoder');
     });
+    it('calls onUserSilence when a speaking burst ends', () => {
+        const { connection, speakingEmitter, streams } = createMockConnection();
+        const onUserSilence = vi.fn();
+        const recv = new AudioReceiver({
+            connection,
+            allowedUserIds: new Set(['111']),
+            sttProvider: createMockStt(),
+            log: createLogger(),
+            createDecoder: createMockDecoder,
+            onUserSilence,
+        });
+        recv.start();
+        speakingEmitter.emit('start', '111');
+        streams.get('111').emit('end');
+        expect(onUserSilence).toHaveBeenCalledWith('111');
+    });
     it('cleans up decoder on stream error', () => {
         const { connection, speakingEmitter, streams } = createMockConnection();
         const decoder = createMockDecoder();

package/dist/voice/conversation-buffer.js

@@ -45,18 +45,28 @@ export class ConversationBuffer {
         }
     }
     /**
-     * Format the stored turns as a conversation log string.
-     * Returns empty string when the buffer is empty.
+     * Return the stored turns from oldest to newest.
+     * Useful for replaying history into providers that support explicit seeding.
      */
-    getHistory() {
+    toTurns() {
         if (this.count === 0)
-            return '';
-        const lines = [];
-        // Read from oldest to newest.
+            return [];
+        const turns = [];
         const start = this.count < CAPACITY ? 0 : this.head;
         for (let i = 0; i < this.count; i++) {
             const idx = (start + i) % CAPACITY;
             const turn = this.buffer[idx];
+            turns.push({ user: turn.user, assistant: turn.assistant });
+        }
+        return turns;
+    }
+    /**
+     * Format the stored turns as a conversation log string.
+     * Returns empty string when the buffer is empty.
+     */
+    getHistory() {
+        const lines = [];
+        for (const turn of this.toTurns()) {
             lines.push(`[User]: ${turn.user}`);
             lines.push(`[Assistant]: ${turn.assistant}`);
         }