@otto-assistant/bridge 0.4.92

package/dist/genai-worker.js

@@ -0,0 +1,311 @@
+// Worker thread for GenAI voice processing.
+// Runs in a separate thread to handle audio encoding/decoding without blocking.
+// Resamples 24kHz GenAI output to 48kHz stereo Opus packets for Discord.
+import { parentPort, threadId } from 'node:worker_threads';
+import { createWriteStream } from 'node:fs';
+import path from 'node:path';
+import * as errore from 'errore';
+import { Resampler } from '@purinton/resampler';
+import * as prism from 'prism-media';
+import { startGenAiSession } from './genai.js';
+import { getTools } from './tools.js';
+import { mkdir } from 'node:fs/promises';
+import { createLogger, formatErrorWithStack, LogPrefix } from './logger.js';
+import { initSentry, notifyError } from './sentry.js';
+if (!parentPort) {
+    throw new Error('This module must be run as a worker thread');
+}
+const workerLogger = createLogger(`${LogPrefix.WORKER}_${threadId}`);
+workerLogger.log('GenAI worker started');
+// Initialize Sentry in worker thread (inherits KIMAKI_SENTRY_DSN from parent)
+initSentry();
+// Define sendError early so it can be used by global handlers
+function sendError(error) {
+    if (parentPort) {
+        parentPort.postMessage({
+            type: 'error',
+            error,
+        });
+    }
+}
+// Add global error handlers for the worker thread
+process.on('uncaughtException', (error) => {
+    workerLogger.error('Uncaught exception in worker:', error);
+    void notifyError(error, 'Uncaught exception in GenAI worker');
+    sendError(`Worker crashed: ${error.message}`);
+    process.exit(1);
+});
+process.on('unhandledRejection', (reason, promise) => {
+    const formattedReason = formatErrorWithStack(reason);
+    workerLogger.error('Unhandled rejection in worker:', formattedReason, 'at promise:', promise);
+    const error = reason instanceof Error
+        ? reason
+        : new Error(formattedReason);
+    void notifyError(error, 'Unhandled rejection in GenAI worker');
+    sendError(`Worker unhandled rejection: ${formattedReason}`);
+});
+// Audio configuration
+const AUDIO_CONFIG = {
+    inputSampleRate: 24000, // GenAI output
+    inputChannels: 1,
+    outputSampleRate: 48000, // Discord expects
+    outputChannels: 2,
+    opusFrameSize: 960, // 20ms at 48kHz
+};
+// Initialize audio processing components
+const resampler = new Resampler({
+    inRate: AUDIO_CONFIG.inputSampleRate,
+    outRate: AUDIO_CONFIG.outputSampleRate,
+    inChannels: AUDIO_CONFIG.inputChannels,
+    outChannels: AUDIO_CONFIG.outputChannels,
+    volume: 1,
+    filterWindow: 8,
+});
+const opusEncoder = new prism.opus.Encoder({
+    rate: AUDIO_CONFIG.outputSampleRate,
+    channels: AUDIO_CONFIG.outputChannels,
+    frameSize: AUDIO_CONFIG.opusFrameSize,
+});
+// Pipe resampler to encoder with error handling
+resampler.pipe(opusEncoder).on('error', (error) => {
+    workerLogger.error('Pipe error between resampler and encoder:', error);
+    void notifyError(error, 'GenAI worker audio pipeline error');
+    sendError(`Audio pipeline error: ${error.message}`);
+});
+// Opus packet queue and interval for 20ms packet sending
+const opusPacketQueue = [];
+let packetInterval = null;
+// Send packets every 20ms
+function startPacketSending() {
+    if (packetInterval)
+        return;
+    packetInterval = setInterval(() => {
+        const packet = opusPacketQueue.shift();
+        if (!packet)
+            return;
+        // Transfer packet as ArrayBuffer
+        const arrayBuffer = packet.buffer.slice(packet.byteOffset, packet.byteOffset + packet.byteLength);
+        parentPort.postMessage({
+            type: 'assistantOpusPacket',
+            packet: arrayBuffer,
+        }, [arrayBuffer]);
+    }, 20);
+}
+function stopPacketSending() {
+    if (packetInterval) {
+        clearInterval(packetInterval);
+        packetInterval = null;
+    }
+    opusPacketQueue.length = 0;
+}
+// Session state
+let session = null;
+// Audio log stream for assistant audio
+let audioLogStream = null;
+// Create assistant audio log stream for debugging
+async function createAssistantAudioLogStream(guildId, channelId) {
+    if (!process.env.DEBUG)
+        return null;
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const audioDir = path.join(process.cwd(), 'discord-audio-logs', guildId, channelId);
+    const mkdirError = await errore.tryAsync({
+        try: () => mkdir(audioDir, { recursive: true }),
+        catch: (e) => e,
+    });
+    if (mkdirError instanceof Error) {
+        workerLogger.error(`Failed to create audio log directory:`, mkdirError.message);
+        return null;
+    }
+    // Create stream for assistant audio (24kHz mono s16le PCM)
+    const outputFileName = `assistant_${timestamp}.24.pcm`;
+    const outputFilePath = path.join(audioDir, outputFileName);
+    const outputAudioStream = createWriteStream(outputFilePath);
+    // Add error handler to prevent crashes
+    outputAudioStream.on('error', (error) => {
+        workerLogger.error(`Assistant audio log stream error:`, error);
+    });
+    workerLogger.log(`Created assistant audio log: ${outputFilePath}`);
+    return outputAudioStream;
+}
+// Handle encoded Opus packets
+opusEncoder.on('data', (packet) => {
+    opusPacketQueue.push(packet);
+});
+// Handle stream end events
+opusEncoder.on('end', () => {
+    workerLogger.log('Opus encoder stream ended');
+});
+resampler.on('end', () => {
+    workerLogger.log('Resampler stream ended');
+});
+// Handle errors
+resampler.on('error', (error) => {
+    workerLogger.error(`Resampler error:`, error);
+    void notifyError(error, 'GenAI worker resampler error');
+    sendError(`Resampler error: ${error.message}`);
+});
+opusEncoder.on('error', (error) => {
+    workerLogger.error(`Encoder error:`, error);
+    const errMsg = error.message || '';
+    // Check for specific corrupted data errors
+    if (errMsg.includes('The compressed data passed is corrupted')) {
+        workerLogger.warn('Received corrupted audio data in opus encoder');
+    }
+    else {
+        void notifyError(error, 'GenAI worker encoder error');
+        sendError(`Encoder error: ${errMsg}`);
+    }
+});
+async function cleanupAsync() {
+    workerLogger.log(`Starting async cleanup`);
+    stopPacketSending();
+    if (session) {
+        workerLogger.log(`Stopping GenAI session`);
+        session.stop();
+        session = null;
+    }
+    // Wait for audio log stream to finish writing
+    if (audioLogStream) {
+        workerLogger.log(`Closing assistant audio log stream`);
+        await new Promise((resolve, reject) => {
+            audioLogStream.end(() => {
+                workerLogger.log(`Assistant audio log stream closed`);
+                resolve();
+            });
+            audioLogStream.on('error', reject);
+            // Add timeout to prevent hanging
+            setTimeout(() => {
+                workerLogger.log(`Audio stream close timeout, continuing`);
+                resolve();
+            }, 3000);
+        });
+        audioLogStream = null;
+    }
+    // Unpipe and end the encoder first
+    resampler.unpipe(opusEncoder);
+    // End the encoder stream
+    await new Promise((resolve) => {
+        opusEncoder.end(() => {
+            workerLogger.log(`Opus encoder ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    // End the resampler stream
+    await new Promise((resolve) => {
+        resampler.end(() => {
+            workerLogger.log(`Resampler ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    workerLogger.log(`Async cleanup complete`);
+}
+// Handle messages from main thread
+parentPort.on('message', async (message) => {
+    try {
+        switch (message.type) {
+            case 'init': {
+                workerLogger.log(`Initializing with directory:`, message.directory);
+                // Create audio log stream for assistant audio
+                audioLogStream = await createAssistantAudioLogStream(message.guildId, message.channelId);
+                // Start packet sending interval
+                startPacketSending();
+                // Get tools for the directory
+                const { tools } = await getTools({
+                    directory: message.directory,
+                    onMessageCompleted: (params) => {
+                        parentPort.postMessage({
+                            type: 'toolCallCompleted',
+                            ...params,
+                        });
+                    },
+                });
+                // Start GenAI session
+                session = await startGenAiSession({
+                    tools,
+                    systemMessage: message.systemMessage,
+                    geminiApiKey: message.geminiApiKey,
+                    onAssistantAudioChunk({ data }) {
+                        // Write to audio log if enabled
+                        if (audioLogStream && !audioLogStream.destroyed) {
+                            audioLogStream.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to audio log:', err);
+                                }
+                            });
+                        }
+                        // Write PCM data to resampler which will output Opus packets
+                        if (!resampler.destroyed) {
+                            resampler.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to resampler:', err);
+                                    sendError(`Failed to process audio: ${err.message}`);
+                                }
+                            });
+                        }
+                    },
+                    onAssistantStartSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStartSpeaking',
+                        });
+                    },
+                    onAssistantStopSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStopSpeaking',
+                        });
+                    },
+                    onAssistantInterruptSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantInterruptSpeaking',
+                        });
+                    },
+                });
+                // Notify main thread we're ready
+                parentPort.postMessage({
+                    type: 'ready',
+                });
+                break;
+            }
+            case 'sendRealtimeInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    audio: message.audio,
+                    audioStreamEnd: message.audioStreamEnd,
+                });
+                break;
+            }
+            case 'sendTextInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    text: message.text,
+                });
+                break;
+            }
+            case 'interrupt': {
+                workerLogger.log(`Interrupting playback`);
+                // Clear the opus packet queue
+                opusPacketQueue.length = 0;
+                break;
+            }
+            case 'stop': {
+                workerLogger.log(`Stopping worker`);
+                await cleanupAsync();
+                // process.exit(0)
+                break;
+            }
+        }
+    }
+    catch (error) {
+        workerLogger.error(`Error handling message:`, error);
+        sendError(error instanceof Error ? error.message : 'Unknown error in worker');
+    }
+});

package/dist/genai.js

@@ -0,0 +1,232 @@
+// Google GenAI Live session manager for real-time voice interactions.
+// Establishes bidirectional audio streaming with Gemini, handles tool calls,
+// and manages the assistant's audio output for Discord voice channels.
+import { GoogleGenAI, LiveServerMessage, MediaResolution, Modality, Session, } from '@google/genai';
+import { writeFile } from 'fs';
+import { createLogger, LogPrefix } from './logger.js';
+import { aiToolToCallableTool } from './ai-tool-to-genai.js';
+const genaiLogger = createLogger(LogPrefix.GENAI);
+const audioParts = [];
+function saveBinaryFile(fileName, content) {
+    writeFile(fileName, content, 'utf8', (err) => {
+        if (err) {
+            genaiLogger.error(`Error writing file ${fileName}:`, err);
+            return;
+        }
+        genaiLogger.log(`Appending stream content to file ${fileName}.`);
+    });
+}
+function convertToWav(rawData, mimeType) {
+    const options = parseMimeType(mimeType);
+    const dataLength = rawData.reduce((a, b) => a + b.length, 0);
+    const wavHeader = createWavHeader(dataLength, options);
+    const buffer = Buffer.concat(rawData);
+    return Buffer.concat([wavHeader, buffer]);
+}
+function parseMimeType(mimeType) {
+    const [fileType, ...params] = mimeType.split(';').map((s) => s.trim());
+    const [_, format] = fileType?.split('/') || [];
+    const options = {
+        numChannels: 1,
+        bitsPerSample: 16,
+    };
+    if (format && format.startsWith('L')) {
+        const bits = parseInt(format.slice(1), 10);
+        if (!isNaN(bits)) {
+            options.bitsPerSample = bits;
+        }
+    }
+    for (const param of params) {
+        const [key, value] = param.split('=').map((s) => s.trim());
+        if (key === 'rate') {
+            options.sampleRate = parseInt(value || '', 10);
+        }
+    }
+    return options;
+}
+function createWavHeader(dataLength, options) {
+    const { numChannels, sampleRate, bitsPerSample } = options;
+    // http://soundfile.sapp.org/doc/WaveFormat
+    const byteRate = (sampleRate * numChannels * bitsPerSample) / 8;
+    const blockAlign = (numChannels * bitsPerSample) / 8;
+    const buffer = Buffer.alloc(44);
+    buffer.write('RIFF', 0); // ChunkID
+    buffer.writeUInt32LE(36 + dataLength, 4); // ChunkSize
+    buffer.write('WAVE', 8); // Format
+    buffer.write('fmt ', 12); // Subchunk1ID
+    buffer.writeUInt32LE(16, 16); // Subchunk1Size (PCM)
+    buffer.writeUInt16LE(1, 20); // AudioFormat (1 = PCM)
+    buffer.writeUInt16LE(numChannels, 22); // NumChannels
+    buffer.writeUInt32LE(sampleRate, 24); // SampleRate
+    buffer.writeUInt32LE(byteRate, 28); // ByteRate
+    buffer.writeUInt16LE(blockAlign, 32); // BlockAlign
+    buffer.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+    buffer.write('data', 36); // Subchunk2ID
+    buffer.writeUInt32LE(dataLength, 40); // Subchunk2Size
+    return buffer;
+}
+function defaultAudioChunkHandler({ data, mimeType, }) {
+    audioParts.push(data);
+    const fileName = 'audio.wav';
+    const buffer = convertToWav(audioParts, mimeType);
+    saveBinaryFile(fileName, buffer);
+}
+export async function startGenAiSession({ onAssistantAudioChunk, onAssistantStartSpeaking, onAssistantStopSpeaking, onAssistantInterruptSpeaking, systemMessage, tools, geminiApiKey, } = {}) {
+    let session = undefined;
+    const callableTools = [];
+    let isAssistantSpeaking = false;
+    const audioChunkHandler = onAssistantAudioChunk || defaultAudioChunkHandler;
+    // Convert AI SDK tools to GenAI CallableTools
+    if (tools) {
+        for (const [name, tool] of Object.entries(tools)) {
+            callableTools.push(aiToolToCallableTool(tool, name));
+        }
+    }
+    function handleModelTurn(message) {
+        if (message.toolCall) {
+            genaiLogger.log('Tool call:', message.toolCall);
+            // Handle tool calls
+            if (message.toolCall.functionCalls && callableTools.length > 0) {
+                for (const tool of callableTools) {
+                    if (!message.toolCall.functionCalls.some((x) => x.name === tool.name)) {
+                        continue;
+                    }
+                    tool
+                        .callTool(message.toolCall.functionCalls)
+                        .then((parts) => {
+                        const functionResponses = parts
+                            .filter((part) => part.functionResponse)
+                            .map((part) => ({
+                            response: part.functionResponse.response,
+                            id: part.functionResponse.id,
+                            name: part.functionResponse.name,
+                        }));
+                        if (functionResponses.length > 0 && session) {
+                            session.sendToolResponse({ functionResponses });
+                            genaiLogger.log('client-toolResponse: ' +
+                                JSON.stringify({ functionResponses }));
+                        }
+                    })
+                        .catch((error) => {
+                        genaiLogger.error('Error handling tool calls:', error);
+                    });
+                }
+            }
+        }
+        if (message.serverContent?.modelTurn?.parts) {
+            for (const part of message.serverContent.modelTurn.parts) {
+                if (part?.fileData) {
+                    genaiLogger.log(`File: ${part?.fileData.fileUri}`);
+                }
+                if (part?.inlineData) {
+                    const inlineData = part.inlineData;
+                    if (!inlineData.mimeType ||
+                        !inlineData.mimeType.startsWith('audio/')) {
+                        genaiLogger.log('Skipping non-audio inlineData:', inlineData.mimeType);
+                        continue;
+                    }
+                    // Trigger start speaking callback the first time audio is received
+                    if (!isAssistantSpeaking && onAssistantStartSpeaking) {
+                        isAssistantSpeaking = true;
+                        onAssistantStartSpeaking();
+                    }
+                    const buffer = Buffer.from(inlineData?.data ?? '', 'base64');
+                    audioChunkHandler({
+                        data: buffer,
+                        mimeType: inlineData.mimeType ?? '',
+                    });
+                }
+                if (part?.text) {
+                    genaiLogger.log('Text:', part.text);
+                }
+            }
+        }
+        // Handle input transcription (user's audio transcription)
+        if (message.serverContent?.inputTranscription?.text) {
+            genaiLogger.log('[user transcription]', message.serverContent.inputTranscription.text);
+        }
+        // Handle output transcription (model's audio transcription)
+        if (message.serverContent?.outputTranscription?.text) {
+            genaiLogger.log('[assistant transcription]', message.serverContent.outputTranscription.text);
+        }
+        if (message.serverContent?.interrupted) {
+            genaiLogger.log('Assistant was interrupted');
+            if (isAssistantSpeaking && onAssistantInterruptSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantInterruptSpeaking();
+            }
+        }
+        if (message.serverContent?.turnComplete) {
+            genaiLogger.log('Assistant turn complete');
+            if (isAssistantSpeaking && onAssistantStopSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantStopSpeaking();
+            }
+        }
+    }
+    const apiKey = geminiApiKey || process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+        genaiLogger.error('No Gemini API key provided');
+        throw new Error('Gemini API key is required for voice interactions');
+    }
+    const ai = new GoogleGenAI({
+        apiKey,
+    });
+    const model = 'gemini-3.1-flash-live-preview';
+    session = await ai.live.connect({
+        model,
+        callbacks: {
+            onopen: function () {
+                genaiLogger.debug('Opened');
+            },
+            onmessage: function (message) {
+                // genaiLogger.log(message)
+                try {
+                    handleModelTurn(message);
+                }
+                catch (error) {
+                    genaiLogger.error('Error handling turn:', error);
+                }
+            },
+            onerror: function (e) {
+                genaiLogger.debug('Error:', e.message);
+            },
+            onclose: function (e) {
+                genaiLogger.debug('Close:', e.reason);
+            },
+        },
+        config: {
+            tools: callableTools,
+            responseModalities: [Modality.AUDIO],
+            mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+            inputAudioTranscription: {}, // transcribes your input speech
+            outputAudioTranscription: {}, // transcribes the model's spoken audio
+            systemInstruction: {
+                parts: [
+                    {
+                        text: systemMessage || '',
+                    },
+                ],
+            },
+            speechConfig: {
+                voiceConfig: {
+                    prebuiltVoiceConfig: {
+                        voiceName: 'Charon', // Orus also not bad
+                    },
+                },
+            },
+            contextWindowCompression: {
+                triggerTokens: '25600',
+                slidingWindow: { targetTokens: '12800' },
+            },
+        },
+    });
+    return {
+        session,
+        stop: () => {
+            const currentSession = session;
+            session = undefined;
+            currentSession?.close();
+        },
+    };
+}

package/dist/generated/browser.js

@@ -0,0 +1,17 @@
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// biome-ignore-all lint: generated file
+// @ts-nocheck 
+/*
+ * This file should be your main import to use Prisma-related types and utilities in a browser.
+ * Use it to get access to models, enums, and input types.
+ *
+ * This file does not contain a `PrismaClient` class, nor several other helpers that are intended as server-side only.
+ * See `client.ts` for the standard, server-side entry point.
+ *
+ * 🟢 You can import this file directly.
+ */
+import * as Prisma from './internal/prismaNamespaceBrowser.js';
+export { Prisma };
+export * as $Enums from './enums.js';
+export * from './enums.js';

package/dist/generated/client.js

@@ -0,0 +1,37 @@
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// biome-ignore-all lint: generated file
+// @ts-nocheck 
+/*
+ * This file should be your main import to use Prisma. Through it you get access to all the models, enums, and input types.
+ * If you're looking for something you can import in the client-side of your application, please refer to the `browser.ts` file instead.
+ *
+ * 🟢 You can import this file directly.
+ */
+import * as process from 'node:process';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+globalThis['__dirname'] = path.dirname(fileURLToPath(import.meta.url));
+import * as runtime from "@prisma/client/runtime/client";
+import * as $Enums from "./enums.js";
+import * as $Class from "./internal/class.js";
+import * as Prisma from "./internal/prismaNamespace.js";
+export * as $Enums from './enums.js';
+export * from "./enums.js";
+/**
+ * ## Prisma Client
+ *
+ * Type-safe database client for TypeScript
+ * @example
+ * ```
+ * const prisma = new PrismaClient({
+ *   adapter: new PrismaPg({ connectionString: process.env.DATABASE_URL })
+ * })
+ * // Fetch zero or more Thread_sessions
+ * const thread_sessions = await prisma.thread_sessions.findMany()
+ * ```
+ *
+ * Read more in our [docs](https://pris.ly/d/client).
+ */
+export const PrismaClient = $Class.getPrismaClientClass();
+export { Prisma };

package/dist/generated/commonInputTypes.js

@@ -0,0 +1,10 @@
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// biome-ignore-all lint: generated file
+// @ts-nocheck 
+/*
+ * This file exports various common sort, input & filter types that are not directly linked to a particular model.
+ *
+ * 🟢 You can import this file directly.
+ */
+import * as $Enums from "./enums.js";

package/dist/generated/enums.js

@@ -0,0 +1,52 @@
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// biome-ignore-all lint: generated file
+// @ts-nocheck 
+/*
+* This file exports all enum related types from the schema.
+*
+* 🟢 You can import this file directly.
+*/
+export const ThreadSessionSource = {
+    kimaki: 'kimaki',
+    external_poll: 'external_poll'
+};
+export const BotMode = {
+    self_hosted: 'self_hosted',
+    gateway: 'gateway'
+};
+export const ChannelType = {
+    text: 'text',
+    voice: 'voice'
+};
+export const WorktreeStatus = {
+    pending: 'pending',
+    ready: 'ready',
+    error: 'error'
+};
+export const VerbosityLevel = {
+    tools_and_text: 'tools_and_text',
+    text_and_essential_tools: 'text_and_essential_tools',
+    text_only: 'text_only'
+};
+export const task_status = {
+    planned: 'planned',
+    running: 'running',
+    completed: 'completed',
+    cancelled: 'cancelled',
+    failed: 'failed'
+};
+export const task_schedule_kind = {
+    at: 'at',
+    cron: 'cron'
+};
+export const ipc_request_type = {
+    file_upload: 'file_upload',
+    action_buttons: 'action_buttons'
+};
+export const ipc_request_status = {
+    pending: 'pending',
+    processing: 'processing',
+    completed: 'completed',
+    cancelled: 'cancelled'
+};