kimaki 0.0.3 → 0.1.0

package/dist/genai-worker-wrapper.js

@@ -0,0 +1,104 @@
+import { Worker } from 'node:worker_threads';
+import { createLogger } from './logger.js';
+const genaiWorkerLogger = createLogger('GENAI WORKER');
+const genaiWrapperLogger = createLogger('GENAI WORKER WRAPPER');
+export function createGenAIWorker(options) {
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(new URL('../dist/genai-worker.js', import.meta.url));
+        // Handle messages from worker
+        worker.on('message', (message) => {
+            switch (message.type) {
+                case 'assistantOpusPacket':
+                    options.onAssistantOpusPacket(message.packet);
+                    break;
+                case 'assistantStartSpeaking':
+                    options.onAssistantStartSpeaking?.();
+                    break;
+                case 'assistantStopSpeaking':
+                    options.onAssistantStopSpeaking?.();
+                    break;
+                case 'assistantInterruptSpeaking':
+                    options.onAssistantInterruptSpeaking?.();
+                    break;
+                case 'toolCallCompleted':
+                    options.onToolCallCompleted?.(message);
+                    break;
+                case 'error':
+                    genaiWorkerLogger.error('Error:', message.error);
+                    options.onError?.(message.error);
+                    break;
+                case 'ready':
+                    genaiWorkerLogger.log('Ready');
+                    // Resolve with the worker interface
+                    resolve({
+                        sendRealtimeInput({ audio, audioStreamEnd }) {
+                            worker.postMessage({
+                                type: 'sendRealtimeInput',
+                                audio,
+                                audioStreamEnd,
+                            });
+                        },
+                        sendTextInput(text) {
+                            worker.postMessage({
+                                type: 'sendTextInput',
+                                text,
+                            });
+                        },
+                        interrupt() {
+                            worker.postMessage({
+                                type: 'interrupt',
+                            });
+                        },
+                        async stop() {
+                            genaiWrapperLogger.log('Stopping worker...');
+                            // Send stop message to trigger graceful shutdown
+                            worker.postMessage({ type: 'stop' });
+                            // Wait for worker to exit gracefully (with timeout)
+                            await new Promise((resolve) => {
+                                let resolved = false;
+                                // Listen for worker exit
+                                worker.once('exit', (code) => {
+                                    if (!resolved) {
+                                        resolved = true;
+                                        genaiWrapperLogger.log(`[GENAI WORKER WRAPPER] Worker exited with code ${code}`);
+                                        resolve();
+                                    }
+                                });
+                                // Timeout after 5 seconds and force terminate
+                                setTimeout(() => {
+                                    if (!resolved) {
+                                        resolved = true;
+                                        genaiWrapperLogger.log('[GENAI WORKER WRAPPER] Worker did not exit gracefully, terminating...');
+                                        worker.terminate().then(() => {
+                                            genaiWrapperLogger.log('Worker terminated');
+                                            resolve();
+                                        });
+                                    }
+                                }, 5000);
+                            });
+                        },
+                    });
+                    break;
+            }
+        });
+        // Handle worker errors
+        worker.on('error', (error) => {
+            genaiWorkerLogger.error('Worker error:', error);
+            reject(error);
+        });
+        worker.on('exit', (code) => {
+            if (code !== 0) {
+                genaiWorkerLogger.error(`Worker stopped with exit code ${code}`);
+            }
+        });
+        // Send initialization message
+        const initMessage = {
+            type: 'init',
+            directory: options.directory,
+            systemMessage: options.systemMessage,
+            guildId: options.guildId,
+            channelId: options.channelId,
+        };
+        worker.postMessage(initMessage);
+    });
+}

package/dist/genai-worker.js

@@ -0,0 +1,293 @@
+import { parentPort, threadId } from 'node:worker_threads';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { Resampler } from '@purinton/resampler';
+import * as prism from 'prism-media';
+import { startGenAiSession } from './genai.js';
+import { getTools } from './tools.js';
+import { createLogger } from './logger.js';
+if (!parentPort) {
+    throw new Error('This module must be run as a worker thread');
+}
+const workerLogger = createLogger(`WORKER ${threadId}`);
+workerLogger.log('GenAI worker started');
+// Define sendError early so it can be used by global handlers
+function sendError(error) {
+    if (parentPort) {
+        parentPort.postMessage({
+            type: 'error',
+            error,
+        });
+    }
+}
+// Add global error handlers for the worker thread
+process.on('uncaughtException', (error) => {
+    workerLogger.error('Uncaught exception in worker:', error);
+    sendError(`Worker crashed: ${error.message}`);
+    // Exit immediately on uncaught exception
+    process.exit(1);
+});
+process.on('unhandledRejection', (reason, promise) => {
+    workerLogger.error('Unhandled rejection in worker:', reason, 'at promise:', promise);
+    sendError(`Worker unhandled rejection: ${reason}`);
+});
+// Audio configuration
+const AUDIO_CONFIG = {
+    inputSampleRate: 24000, // GenAI output
+    inputChannels: 1,
+    outputSampleRate: 48000, // Discord expects
+    outputChannels: 2,
+    opusFrameSize: 960, // 20ms at 48kHz
+};
+// Initialize audio processing components
+const resampler = new Resampler({
+    inRate: AUDIO_CONFIG.inputSampleRate,
+    outRate: AUDIO_CONFIG.outputSampleRate,
+    inChannels: AUDIO_CONFIG.inputChannels,
+    outChannels: AUDIO_CONFIG.outputChannels,
+    volume: 1,
+    filterWindow: 8,
+});
+const opusEncoder = new prism.opus.Encoder({
+    rate: AUDIO_CONFIG.outputSampleRate,
+    channels: AUDIO_CONFIG.outputChannels,
+    frameSize: AUDIO_CONFIG.opusFrameSize,
+});
+// Pipe resampler to encoder with error handling
+resampler.pipe(opusEncoder).on('error', (error) => {
+    workerLogger.error('Pipe error between resampler and encoder:', error);
+    sendError(`Audio pipeline error: ${error.message}`);
+});
+// Opus packet queue and interval for 20ms packet sending
+const opusPacketQueue = [];
+let packetInterval = null;
+// Send packets every 20ms
+function startPacketSending() {
+    if (packetInterval)
+        return;
+    packetInterval = setInterval(() => {
+        const packet = opusPacketQueue.shift();
+        if (!packet)
+            return;
+        // Transfer packet as ArrayBuffer
+        const arrayBuffer = packet.buffer.slice(packet.byteOffset, packet.byteOffset + packet.byteLength);
+        parentPort.postMessage({
+            type: 'assistantOpusPacket',
+            packet: arrayBuffer,
+        }, [arrayBuffer]);
+    }, 20);
+}
+function stopPacketSending() {
+    if (packetInterval) {
+        clearInterval(packetInterval);
+        packetInterval = null;
+    }
+    opusPacketQueue.length = 0;
+}
+// Session state
+let session = null;
+// Audio log stream for assistant audio
+let audioLogStream = null;
+// Create assistant audio log stream for debugging
+async function createAssistantAudioLogStream(guildId, channelId) {
+    if (!process.env.DEBUG)
+        return null;
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const audioDir = path.join(process.cwd(), 'discord-audio-logs', guildId, channelId);
+    try {
+        await mkdir(audioDir, { recursive: true });
+        // Create stream for assistant audio (24kHz mono s16le PCM)
+        const outputFileName = `assistant_${timestamp}.24.pcm`;
+        const outputFilePath = path.join(audioDir, outputFileName);
+        const outputAudioStream = createWriteStream(outputFilePath);
+        // Add error handler to prevent crashes
+        outputAudioStream.on('error', (error) => {
+            workerLogger.error(`Assistant audio log stream error:`, error);
+        });
+        workerLogger.log(`Created assistant audio log: ${outputFilePath}`);
+        return outputAudioStream;
+    }
+    catch (error) {
+        workerLogger.error(`Failed to create audio log directory:`, error);
+        return null;
+    }
+}
+// Handle encoded Opus packets
+opusEncoder.on('data', (packet) => {
+    opusPacketQueue.push(packet);
+});
+// Handle stream end events
+opusEncoder.on('end', () => {
+    workerLogger.log('Opus encoder stream ended');
+});
+resampler.on('end', () => {
+    workerLogger.log('Resampler stream ended');
+});
+// Handle errors
+resampler.on('error', (error) => {
+    workerLogger.error(`Resampler error:`, error);
+    sendError(`Resampler error: ${error.message}`);
+});
+opusEncoder.on('error', (error) => {
+    workerLogger.error(`Encoder error:`, error);
+    // Check for specific corrupted data errors
+    if (error.message?.includes('The compressed data passed is corrupted')) {
+        workerLogger.warn('Received corrupted audio data in opus encoder');
+    }
+    else {
+        sendError(`Encoder error: ${error.message}`);
+    }
+});
+async function cleanupAsync() {
+    workerLogger.log(`Starting async cleanup`);
+    stopPacketSending();
+    if (session) {
+        workerLogger.log(`Stopping GenAI session`);
+        session.stop();
+        session = null;
+    }
+    // Wait for audio log stream to finish writing
+    if (audioLogStream) {
+        workerLogger.log(`Closing assistant audio log stream`);
+        await new Promise((resolve, reject) => {
+            audioLogStream.end(() => {
+                workerLogger.log(`Assistant audio log stream closed`);
+                resolve();
+            });
+            audioLogStream.on('error', reject);
+            // Add timeout to prevent hanging
+            setTimeout(() => {
+                workerLogger.log(`Audio stream close timeout, continuing`);
+                resolve();
+            }, 3000);
+        });
+        audioLogStream = null;
+    }
+    // Unpipe and end the encoder first
+    resampler.unpipe(opusEncoder);
+    // End the encoder stream
+    await new Promise((resolve) => {
+        opusEncoder.end(() => {
+            workerLogger.log(`Opus encoder ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    // End the resampler stream
+    await new Promise((resolve) => {
+        resampler.end(() => {
+            workerLogger.log(`Resampler ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    workerLogger.log(`Async cleanup complete`);
+}
+// Handle messages from main thread
+parentPort.on('message', async (message) => {
+    try {
+        switch (message.type) {
+            case 'init': {
+                workerLogger.log(`Initializing with directory:`, message.directory);
+                // Create audio log stream for assistant audio
+                audioLogStream = await createAssistantAudioLogStream(message.guildId, message.channelId);
+                // Start packet sending interval
+                startPacketSending();
+                // Get tools for the directory
+                const { tools } = await getTools({
+                    directory: message.directory,
+                    onMessageCompleted: (params) => {
+                        parentPort.postMessage({
+                            type: 'toolCallCompleted',
+                            ...params,
+                        });
+                    },
+                });
+                // Start GenAI session
+                session = await startGenAiSession({
+                    tools,
+                    systemMessage: message.systemMessage,
+                    onAssistantAudioChunk({ data }) {
+                        // Write to audio log if enabled
+                        if (audioLogStream && !audioLogStream.destroyed) {
+                            audioLogStream.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to audio log:', err);
+                                }
+                            });
+                        }
+                        // Write PCM data to resampler which will output Opus packets
+                        if (!resampler.destroyed) {
+                            resampler.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to resampler:', err);
+                                    sendError(`Failed to process audio: ${err.message}`);
+                                }
+                            });
+                        }
+                    },
+                    onAssistantStartSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStartSpeaking',
+                        });
+                    },
+                    onAssistantStopSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStopSpeaking',
+                        });
+                    },
+                    onAssistantInterruptSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantInterruptSpeaking',
+                        });
+                    },
+                });
+                // Notify main thread we're ready
+                parentPort.postMessage({
+                    type: 'ready',
+                });
+                break;
+            }
+            case 'sendRealtimeInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    audio: message.audio,
+                    audioStreamEnd: message.audioStreamEnd,
+                });
+                break;
+            }
+            case 'sendTextInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    text: message.text,
+                });
+                break;
+            }
+            case 'interrupt': {
+                workerLogger.log(`Interrupting playback`);
+                // Clear the opus packet queue
+                opusPacketQueue.length = 0;
+                break;
+            }
+            case 'stop': {
+                workerLogger.log(`Stopping worker`);
+                await cleanupAsync();
+                // process.exit(0)
+                break;
+            }
+        }
+    }
+    catch (error) {
+        workerLogger.error(`Error handling message:`, error);
+        sendError(error instanceof Error ? error.message : 'Unknown error in worker');
+    }
+});

package/dist/genai.js

@@ -0,0 +1,224 @@
+import { GoogleGenAI, LiveServerMessage, MediaResolution, Modality, Session, } from '@google/genai';
+import { writeFile } from 'fs';
+import { createLogger } from './logger.js';
+import { aiToolToCallableTool } from './ai-tool-to-genai.js';
+const genaiLogger = createLogger('GENAI');
+const audioParts = [];
+function saveBinaryFile(fileName, content) {
+    writeFile(fileName, content, 'utf8', (err) => {
+        if (err) {
+            genaiLogger.error(`Error writing file ${fileName}:`, err);
+            return;
+        }
+        genaiLogger.log(`Appending stream content to file ${fileName}.`);
+    });
+}
+function convertToWav(rawData, mimeType) {
+    const options = parseMimeType(mimeType);
+    const dataLength = rawData.reduce((a, b) => a + b.length, 0);
+    const wavHeader = createWavHeader(dataLength, options);
+    const buffer = Buffer.concat(rawData);
+    return Buffer.concat([wavHeader, buffer]);
+}
+function parseMimeType(mimeType) {
+    const [fileType, ...params] = mimeType.split(';').map((s) => s.trim());
+    const [_, format] = fileType?.split('/') || [];
+    const options = {
+        numChannels: 1,
+        bitsPerSample: 16,
+    };
+    if (format && format.startsWith('L')) {
+        const bits = parseInt(format.slice(1), 10);
+        if (!isNaN(bits)) {
+            options.bitsPerSample = bits;
+        }
+    }
+    for (const param of params) {
+        const [key, value] = param.split('=').map((s) => s.trim());
+        if (key === 'rate') {
+            options.sampleRate = parseInt(value || '', 10);
+        }
+    }
+    return options;
+}
+function createWavHeader(dataLength, options) {
+    const { numChannels, sampleRate, bitsPerSample } = options;
+    // http://soundfile.sapp.org/doc/WaveFormat
+    const byteRate = (sampleRate * numChannels * bitsPerSample) / 8;
+    const blockAlign = (numChannels * bitsPerSample) / 8;
+    const buffer = Buffer.alloc(44);
+    buffer.write('RIFF', 0); // ChunkID
+    buffer.writeUInt32LE(36 + dataLength, 4); // ChunkSize
+    buffer.write('WAVE', 8); // Format
+    buffer.write('fmt ', 12); // Subchunk1ID
+    buffer.writeUInt32LE(16, 16); // Subchunk1Size (PCM)
+    buffer.writeUInt16LE(1, 20); // AudioFormat (1 = PCM)
+    buffer.writeUInt16LE(numChannels, 22); // NumChannels
+    buffer.writeUInt32LE(sampleRate, 24); // SampleRate
+    buffer.writeUInt32LE(byteRate, 28); // ByteRate
+    buffer.writeUInt16LE(blockAlign, 32); // BlockAlign
+    buffer.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+    buffer.write('data', 36); // Subchunk2ID
+    buffer.writeUInt32LE(dataLength, 40); // Subchunk2Size
+    return buffer;
+}
+function defaultAudioChunkHandler({ data, mimeType, }) {
+    audioParts.push(data);
+    const fileName = 'audio.wav';
+    const buffer = convertToWav(audioParts, mimeType);
+    saveBinaryFile(fileName, buffer);
+}
+export async function startGenAiSession({ onAssistantAudioChunk, onAssistantStartSpeaking, onAssistantStopSpeaking, onAssistantInterruptSpeaking, systemMessage, tools, } = {}) {
+    let session = undefined;
+    const callableTools = [];
+    let isAssistantSpeaking = false;
+    const audioChunkHandler = onAssistantAudioChunk || defaultAudioChunkHandler;
+    // Convert AI SDK tools to GenAI CallableTools
+    if (tools) {
+        for (const [name, tool] of Object.entries(tools)) {
+            callableTools.push(aiToolToCallableTool(tool, name));
+        }
+    }
+    function handleModelTurn(message) {
+        if (message.toolCall) {
+            genaiLogger.log('Tool call:', message.toolCall);
+            // Handle tool calls
+            if (message.toolCall.functionCalls && callableTools.length > 0) {
+                for (const tool of callableTools) {
+                    if (!message.toolCall.functionCalls.some((x) => x.name === tool.name)) {
+                        continue;
+                    }
+                    tool
+                        .callTool(message.toolCall.functionCalls)
+                        .then((parts) => {
+                        const functionResponses = parts
+                            .filter((part) => part.functionResponse)
+                            .map((part) => ({
+                            response: part.functionResponse.response,
+                            id: part.functionResponse.id,
+                            name: part.functionResponse.name,
+                        }));
+                        if (functionResponses.length > 0 && session) {
+                            session.sendToolResponse({ functionResponses });
+                            genaiLogger.log('client-toolResponse: ' +
+                                JSON.stringify({ functionResponses }));
+                        }
+                    })
+                        .catch((error) => {
+                        genaiLogger.error('Error handling tool calls:', error);
+                    });
+                }
+            }
+        }
+        if (message.serverContent?.modelTurn?.parts) {
+            for (const part of message.serverContent.modelTurn.parts) {
+                if (part?.fileData) {
+                    genaiLogger.log(`File: ${part?.fileData.fileUri}`);
+                }
+                if (part?.inlineData) {
+                    const inlineData = part.inlineData;
+                    if (!inlineData.mimeType ||
+                        !inlineData.mimeType.startsWith('audio/')) {
+                        genaiLogger.log('Skipping non-audio inlineData:', inlineData.mimeType);
+                        continue;
+                    }
+                    // Trigger start speaking callback the first time audio is received
+                    if (!isAssistantSpeaking && onAssistantStartSpeaking) {
+                        isAssistantSpeaking = true;
+                        onAssistantStartSpeaking();
+                    }
+                    const buffer = Buffer.from(inlineData?.data ?? '', 'base64');
+                    audioChunkHandler({
+                        data: buffer,
+                        mimeType: inlineData.mimeType ?? '',
+                    });
+                }
+                if (part?.text) {
+                    genaiLogger.log('Text:', part.text);
+                }
+            }
+        }
+        // Handle input transcription (user's audio transcription)
+        if (message.serverContent?.inputTranscription?.text) {
+            genaiLogger.log('[user transcription]', message.serverContent.inputTranscription.text);
+        }
+        // Handle output transcription (model's audio transcription)
+        if (message.serverContent?.outputTranscription?.text) {
+            genaiLogger.log('[assistant transcription]', message.serverContent.outputTranscription.text);
+        }
+        if (message.serverContent?.interrupted) {
+            genaiLogger.log('Assistant was interrupted');
+            if (isAssistantSpeaking && onAssistantInterruptSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantInterruptSpeaking();
+            }
+        }
+        if (message.serverContent?.turnComplete) {
+            genaiLogger.log('Assistant turn complete');
+            if (isAssistantSpeaking && onAssistantStopSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantStopSpeaking();
+            }
+        }
+    }
+    const ai = new GoogleGenAI({
+        apiKey: process.env.GEMINI_API_KEY,
+    });
+    const model = 'models/gemini-2.5-flash-live-preview';
+    session = await ai.live.connect({
+        model,
+        callbacks: {
+            onopen: function () {
+                genaiLogger.debug('Opened');
+            },
+            onmessage: function (message) {
+                // genaiLogger.log(message)
+                try {
+                    handleModelTurn(message);
+                }
+                catch (error) {
+                    genaiLogger.error('Error handling turn:', error);
+                }
+            },
+            onerror: function (e) {
+                genaiLogger.debug('Error:', e.message);
+            },
+            onclose: function (e) {
+                genaiLogger.debug('Close:', e.reason);
+            },
+        },
+        config: {
+            tools: callableTools,
+            responseModalities: [Modality.AUDIO],
+            mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+            inputAudioTranscription: {}, // transcribes your input speech
+            outputAudioTranscription: {}, // transcribes the model's spoken audio
+            systemInstruction: {
+                parts: [
+                    {
+                        text: systemMessage || '',
+                    },
+                ],
+            },
+            speechConfig: {
+                voiceConfig: {
+                    prebuiltVoiceConfig: {
+                        voiceName: 'Charon', // Orus also not bad
+                    },
+                },
+            },
+            contextWindowCompression: {
+                triggerTokens: '25600',
+                slidingWindow: { targetTokens: '12800' },
+            },
+        },
+    });
+    return {
+        session,
+        stop: () => {
+            const currentSession = session;
+            session = undefined;
+            currentSession?.close();
+        },
+    };
+}

package/dist/logger.js

@@ -0,0 +1,10 @@
+import { log } from '@clack/prompts';
+export function createLogger(prefix) {
+    return {
+        log: (...args) => log.info([`[${prefix}]`, ...args.map((arg) => String(arg))].join(' ')),
+        error: (...args) => log.error([`[${prefix}]`, ...args.map((arg) => String(arg))].join(' ')),
+        warn: (...args) => log.warn([`[${prefix}]`, ...args.map((arg) => String(arg))].join(' ')),
+        info: (...args) => log.info([`[${prefix}]`, ...args.map((arg) => String(arg))].join(' ')),
+        debug: (...args) => log.info([`[${prefix}]`, ...args.map((arg) => String(arg))].join(' ')),
+    };
+}