disunday 1.0.0

package/dist/genai-worker.js

@@ -0,0 +1,299 @@
+// Worker thread for GenAI voice processing.
+// Runs in a separate thread to handle audio encoding/decoding without blocking.
+// Resamples 24kHz GenAI output to 48kHz stereo Opus packets for Discord.
+import { parentPort, threadId } from 'node:worker_threads';
+import { createWriteStream } from 'node:fs';
+import path from 'node:path';
+import * as errore from 'errore';
+import { Resampler } from '@purinton/resampler';
+import * as prism from 'prism-media';
+import { startGenAiSession } from './genai.js';
+import { getTools } from './tools.js';
+import { mkdir } from 'node:fs/promises';
+import { createLogger, LogPrefix } from './logger.js';
+if (!parentPort) {
+    throw new Error('This module must be run as a worker thread');
+}
+const workerLogger = createLogger(`${LogPrefix.WORKER}_${threadId}`);
+workerLogger.log('GenAI worker started');
+// Define sendError early so it can be used by global handlers
+function sendError(error) {
+    if (parentPort) {
+        parentPort.postMessage({
+            type: 'error',
+            error,
+        });
+    }
+}
+// Add global error handlers for the worker thread
+process.on('uncaughtException', (error) => {
+    workerLogger.error('Uncaught exception in worker:', error);
+    sendError(`Worker crashed: ${error.message}`);
+    // Exit immediately on uncaught exception
+    process.exit(1);
+});
+process.on('unhandledRejection', (reason, promise) => {
+    workerLogger.error('Unhandled rejection in worker:', reason, 'at promise:', promise);
+    sendError(`Worker unhandled rejection: ${reason}`);
+});
+// Audio configuration
+const AUDIO_CONFIG = {
+    inputSampleRate: 24000, // GenAI output
+    inputChannels: 1,
+    outputSampleRate: 48000, // Discord expects
+    outputChannels: 2,
+    opusFrameSize: 960, // 20ms at 48kHz
+};
+// Initialize audio processing components
+const resampler = new Resampler({
+    inRate: AUDIO_CONFIG.inputSampleRate,
+    outRate: AUDIO_CONFIG.outputSampleRate,
+    inChannels: AUDIO_CONFIG.inputChannels,
+    outChannels: AUDIO_CONFIG.outputChannels,
+    volume: 1,
+    filterWindow: 8,
+});
+const opusEncoder = new prism.opus.Encoder({
+    rate: AUDIO_CONFIG.outputSampleRate,
+    channels: AUDIO_CONFIG.outputChannels,
+    frameSize: AUDIO_CONFIG.opusFrameSize,
+});
+// Pipe resampler to encoder with error handling
+resampler.pipe(opusEncoder).on('error', (error) => {
+    workerLogger.error('Pipe error between resampler and encoder:', error);
+    sendError(`Audio pipeline error: ${error.message}`);
+});
+// Opus packet queue and interval for 20ms packet sending
+const opusPacketQueue = [];
+let packetInterval = null;
+// Send packets every 20ms
+function startPacketSending() {
+    if (packetInterval)
+        return;
+    packetInterval = setInterval(() => {
+        const packet = opusPacketQueue.shift();
+        if (!packet)
+            return;
+        // Transfer packet as ArrayBuffer
+        const arrayBuffer = packet.buffer.slice(packet.byteOffset, packet.byteOffset + packet.byteLength);
+        parentPort.postMessage({
+            type: 'assistantOpusPacket',
+            packet: arrayBuffer,
+        }, [arrayBuffer]);
+    }, 20);
+}
+function stopPacketSending() {
+    if (packetInterval) {
+        clearInterval(packetInterval);
+        packetInterval = null;
+    }
+    opusPacketQueue.length = 0;
+}
+// Session state
+let session = null;
+// Audio log stream for assistant audio
+let audioLogStream = null;
+// Create assistant audio log stream for debugging
+async function createAssistantAudioLogStream(guildId, channelId) {
+    if (!process.env.DEBUG)
+        return null;
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const audioDir = path.join(process.cwd(), 'discord-audio-logs', guildId, channelId);
+    const mkdirError = await errore.tryAsync({
+        try: () => mkdir(audioDir, { recursive: true }),
+        catch: (e) => e,
+    });
+    if (mkdirError instanceof Error) {
+        workerLogger.error(`Failed to create audio log directory:`, mkdirError.message);
+        return null;
+    }
+    // Create stream for assistant audio (24kHz mono s16le PCM)
+    const outputFileName = `assistant_${timestamp}.24.pcm`;
+    const outputFilePath = path.join(audioDir, outputFileName);
+    const outputAudioStream = createWriteStream(outputFilePath);
+    // Add error handler to prevent crashes
+    outputAudioStream.on('error', (error) => {
+        workerLogger.error(`Assistant audio log stream error:`, error);
+    });
+    workerLogger.log(`Created assistant audio log: ${outputFilePath}`);
+    return outputAudioStream;
+}
+// Handle encoded Opus packets
+opusEncoder.on('data', (packet) => {
+    opusPacketQueue.push(packet);
+});
+// Handle stream end events
+opusEncoder.on('end', () => {
+    workerLogger.log('Opus encoder stream ended');
+});
+resampler.on('end', () => {
+    workerLogger.log('Resampler stream ended');
+});
+// Handle errors
+resampler.on('error', (error) => {
+    workerLogger.error(`Resampler error:`, error);
+    sendError(`Resampler error: ${error.message}`);
+});
+opusEncoder.on('error', (error) => {
+    workerLogger.error(`Encoder error:`, error);
+    // Check for specific corrupted data errors
+    if (error.message?.includes('The compressed data passed is corrupted')) {
+        workerLogger.warn('Received corrupted audio data in opus encoder');
+    }
+    else {
+        sendError(`Encoder error: ${error.message}`);
+    }
+});
+async function cleanupAsync() {
+    workerLogger.log(`Starting async cleanup`);
+    stopPacketSending();
+    if (session) {
+        workerLogger.log(`Stopping GenAI session`);
+        session.stop();
+        session = null;
+    }
+    // Wait for audio log stream to finish writing
+    if (audioLogStream) {
+        workerLogger.log(`Closing assistant audio log stream`);
+        await new Promise((resolve, reject) => {
+            audioLogStream.end(() => {
+                workerLogger.log(`Assistant audio log stream closed`);
+                resolve();
+            });
+            audioLogStream.on('error', reject);
+            // Add timeout to prevent hanging
+            setTimeout(() => {
+                workerLogger.log(`Audio stream close timeout, continuing`);
+                resolve();
+            }, 3000);
+        });
+        audioLogStream = null;
+    }
+    // Unpipe and end the encoder first
+    resampler.unpipe(opusEncoder);
+    // End the encoder stream
+    await new Promise((resolve) => {
+        opusEncoder.end(() => {
+            workerLogger.log(`Opus encoder ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    // End the resampler stream
+    await new Promise((resolve) => {
+        resampler.end(() => {
+            workerLogger.log(`Resampler ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    workerLogger.log(`Async cleanup complete`);
+}
+// Handle messages from main thread
+parentPort.on('message', async (message) => {
+    try {
+        switch (message.type) {
+            case 'init': {
+                workerLogger.log(`Initializing with directory:`, message.directory);
+                // Create audio log stream for assistant audio
+                audioLogStream = await createAssistantAudioLogStream(message.guildId, message.channelId);
+                // Start packet sending interval
+                startPacketSending();
+                // Get tools for the directory
+                const { tools } = await getTools({
+                    directory: message.directory,
+                    onMessageCompleted: (params) => {
+                        parentPort.postMessage({
+                            type: 'toolCallCompleted',
+                            ...params,
+                        });
+                    },
+                });
+                // Start GenAI session
+                session = await startGenAiSession({
+                    tools,
+                    systemMessage: message.systemMessage,
+                    geminiApiKey: message.geminiApiKey,
+                    onAssistantAudioChunk({ data }) {
+                        // Write to audio log if enabled
+                        if (audioLogStream && !audioLogStream.destroyed) {
+                            audioLogStream.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to audio log:', err);
+                                }
+                            });
+                        }
+                        // Write PCM data to resampler which will output Opus packets
+                        if (!resampler.destroyed) {
+                            resampler.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to resampler:', err);
+                                    sendError(`Failed to process audio: ${err.message}`);
+                                }
+                            });
+                        }
+                    },
+                    onAssistantStartSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStartSpeaking',
+                        });
+                    },
+                    onAssistantStopSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStopSpeaking',
+                        });
+                    },
+                    onAssistantInterruptSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantInterruptSpeaking',
+                        });
+                    },
+                });
+                // Notify main thread we're ready
+                parentPort.postMessage({
+                    type: 'ready',
+                });
+                break;
+            }
+            case 'sendRealtimeInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    audio: message.audio,
+                    audioStreamEnd: message.audioStreamEnd,
+                });
+                break;
+            }
+            case 'sendTextInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    text: message.text,
+                });
+                break;
+            }
+            case 'interrupt': {
+                workerLogger.log(`Interrupting playback`);
+                // Clear the opus packet queue
+                opusPacketQueue.length = 0;
+                break;
+            }
+            case 'stop': {
+                workerLogger.log(`Stopping worker`);
+                await cleanupAsync();
+                // process.exit(0)
+                break;
+            }
+        }
+    }
+    catch (error) {
+        workerLogger.error(`Error handling message:`, error);
+        sendError(error instanceof Error ? error.message : 'Unknown error in worker');
+    }
+});

package/dist/genai.js

@@ -0,0 +1,230 @@
+// Google GenAI Live session manager for real-time voice interactions.
+// Establishes bidirectional audio streaming with Gemini, handles tool calls,
+// and manages the assistant's audio output for Discord voice channels.
+import { GoogleGenAI, LiveServerMessage, MediaResolution, Modality, Session } from '@google/genai';
+import { writeFile } from 'fs';
+import { createLogger, LogPrefix } from './logger.js';
+import { aiToolToCallableTool } from './ai-tool-to-genai.js';
+const genaiLogger = createLogger(LogPrefix.GENAI);
+const audioParts = [];
+function saveBinaryFile(fileName, content) {
+    writeFile(fileName, content, 'utf8', (err) => {
+        if (err) {
+            genaiLogger.error(`Error writing file ${fileName}:`, err);
+            return;
+        }
+        genaiLogger.log(`Appending stream content to file ${fileName}.`);
+    });
+}
+function convertToWav(rawData, mimeType) {
+    const options = parseMimeType(mimeType);
+    const dataLength = rawData.reduce((a, b) => a + b.length, 0);
+    const wavHeader = createWavHeader(dataLength, options);
+    const buffer = Buffer.concat(rawData);
+    return Buffer.concat([wavHeader, buffer]);
+}
+function parseMimeType(mimeType) {
+    const [fileType, ...params] = mimeType.split(';').map((s) => s.trim());
+    const [_, format] = fileType?.split('/') || [];
+    const options = {
+        numChannels: 1,
+        bitsPerSample: 16,
+    };
+    if (format && format.startsWith('L')) {
+        const bits = parseInt(format.slice(1), 10);
+        if (!isNaN(bits)) {
+            options.bitsPerSample = bits;
+        }
+    }
+    for (const param of params) {
+        const [key, value] = param.split('=').map((s) => s.trim());
+        if (key === 'rate') {
+            options.sampleRate = parseInt(value || '', 10);
+        }
+    }
+    return options;
+}
+function createWavHeader(dataLength, options) {
+    const { numChannels, sampleRate, bitsPerSample } = options;
+    // http://soundfile.sapp.org/doc/WaveFormat
+    const byteRate = (sampleRate * numChannels * bitsPerSample) / 8;
+    const blockAlign = (numChannels * bitsPerSample) / 8;
+    const buffer = Buffer.alloc(44);
+    buffer.write('RIFF', 0); // ChunkID
+    buffer.writeUInt32LE(36 + dataLength, 4); // ChunkSize
+    buffer.write('WAVE', 8); // Format
+    buffer.write('fmt ', 12); // Subchunk1ID
+    buffer.writeUInt32LE(16, 16); // Subchunk1Size (PCM)
+    buffer.writeUInt16LE(1, 20); // AudioFormat (1 = PCM)
+    buffer.writeUInt16LE(numChannels, 22); // NumChannels
+    buffer.writeUInt32LE(sampleRate, 24); // SampleRate
+    buffer.writeUInt32LE(byteRate, 28); // ByteRate
+    buffer.writeUInt16LE(blockAlign, 32); // BlockAlign
+    buffer.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+    buffer.write('data', 36); // Subchunk2ID
+    buffer.writeUInt32LE(dataLength, 40); // Subchunk2Size
+    return buffer;
+}
+function defaultAudioChunkHandler({ data, mimeType }) {
+    audioParts.push(data);
+    const fileName = 'audio.wav';
+    const buffer = convertToWav(audioParts, mimeType);
+    saveBinaryFile(fileName, buffer);
+}
+export async function startGenAiSession({ onAssistantAudioChunk, onAssistantStartSpeaking, onAssistantStopSpeaking, onAssistantInterruptSpeaking, systemMessage, tools, geminiApiKey, } = {}) {
+    let session = undefined;
+    const callableTools = [];
+    let isAssistantSpeaking = false;
+    const audioChunkHandler = onAssistantAudioChunk || defaultAudioChunkHandler;
+    // Convert AI SDK tools to GenAI CallableTools
+    if (tools) {
+        for (const [name, tool] of Object.entries(tools)) {
+            callableTools.push(aiToolToCallableTool(tool, name));
+        }
+    }
+    function handleModelTurn(message) {
+        if (message.toolCall) {
+            genaiLogger.log('Tool call:', message.toolCall);
+            // Handle tool calls
+            if (message.toolCall.functionCalls && callableTools.length > 0) {
+                for (const tool of callableTools) {
+                    if (!message.toolCall.functionCalls.some((x) => x.name === tool.name)) {
+                        continue;
+                    }
+                    tool
+                        .callTool(message.toolCall.functionCalls)
+                        .then((parts) => {
+                        const functionResponses = parts
+                            .filter((part) => part.functionResponse)
+                            .map((part) => ({
+                            response: part.functionResponse.response,
+                            id: part.functionResponse.id,
+                            name: part.functionResponse.name,
+                        }));
+                        if (functionResponses.length > 0 && session) {
+                            session.sendToolResponse({ functionResponses });
+                            genaiLogger.log('client-toolResponse: ' + JSON.stringify({ functionResponses }));
+                        }
+                    })
+                        .catch((error) => {
+                        genaiLogger.error('Error handling tool calls:', error);
+                    });
+                }
+            }
+        }
+        if (message.serverContent?.modelTurn?.parts) {
+            for (const part of message.serverContent.modelTurn.parts) {
+                if (part?.fileData) {
+                    genaiLogger.log(`File: ${part?.fileData.fileUri}`);
+                }
+                if (part?.inlineData) {
+                    const inlineData = part.inlineData;
+                    if (!inlineData.mimeType || !inlineData.mimeType.startsWith('audio/')) {
+                        genaiLogger.log('Skipping non-audio inlineData:', inlineData.mimeType);
+                        continue;
+                    }
+                    // Trigger start speaking callback the first time audio is received
+                    if (!isAssistantSpeaking && onAssistantStartSpeaking) {
+                        isAssistantSpeaking = true;
+                        onAssistantStartSpeaking();
+                    }
+                    const buffer = Buffer.from(inlineData?.data ?? '', 'base64');
+                    audioChunkHandler({
+                        data: buffer,
+                        mimeType: inlineData.mimeType ?? '',
+                    });
+                }
+                if (part?.text) {
+                    genaiLogger.log('Text:', part.text);
+                }
+            }
+        }
+        // Handle input transcription (user's audio transcription)
+        if (message.serverContent?.inputTranscription?.text) {
+            genaiLogger.log('[user transcription]', message.serverContent.inputTranscription.text);
+        }
+        // Handle output transcription (model's audio transcription)
+        if (message.serverContent?.outputTranscription?.text) {
+            genaiLogger.log('[assistant transcription]', message.serverContent.outputTranscription.text);
+        }
+        if (message.serverContent?.interrupted) {
+            genaiLogger.log('Assistant was interrupted');
+            if (isAssistantSpeaking && onAssistantInterruptSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantInterruptSpeaking();
+            }
+        }
+        if (message.serverContent?.turnComplete) {
+            genaiLogger.log('Assistant turn complete');
+            if (isAssistantSpeaking && onAssistantStopSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantStopSpeaking();
+            }
+        }
+    }
+    const apiKey = geminiApiKey || process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+        genaiLogger.error('No Gemini API key provided');
+        throw new Error('Gemini API key is required for voice interactions');
+    }
+    const ai = new GoogleGenAI({
+        apiKey,
+    });
+    const model = 'gemini-2.5-flash-native-audio-preview-12-2025';
+    session = await ai.live.connect({
+        model,
+        callbacks: {
+            onopen: function () {
+                genaiLogger.debug('Opened');
+            },
+            onmessage: function (message) {
+                // genaiLogger.log(message)
+                try {
+                    handleModelTurn(message);
+                }
+                catch (error) {
+                    genaiLogger.error('Error handling turn:', error);
+                }
+            },
+            onerror: function (e) {
+                genaiLogger.debug('Error:', e.message);
+            },
+            onclose: function (e) {
+                genaiLogger.debug('Close:', e.reason);
+            },
+        },
+        config: {
+            tools: callableTools,
+            responseModalities: [Modality.AUDIO],
+            mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+            inputAudioTranscription: {}, // transcribes your input speech
+            outputAudioTranscription: {}, // transcribes the model's spoken audio
+            systemInstruction: {
+                parts: [
+                    {
+                        text: systemMessage || '',
+                    },
+                ],
+            },
+            speechConfig: {
+                voiceConfig: {
+                    prebuiltVoiceConfig: {
+                        voiceName: 'Charon', // Orus also not bad
+                    },
+                },
+            },
+            contextWindowCompression: {
+                triggerTokens: '25600',
+                slidingWindow: { targetTokens: '12800' },
+            },
+        },
+    });
+    return {
+        session,
+        stop: () => {
+            const currentSession = session;
+            session = undefined;
+            currentSession?.close();
+        },
+    };
+}

package/dist/image-utils.js

@@ -0,0 +1,107 @@
+// Image processing utilities for Discord attachments.
+// Uses sharp (optional) to resize large images and heic-convert (optional) for HEIC support.
+// Falls back gracefully if dependencies are not available.
+import { createLogger, LogPrefix } from './logger.js';
+const logger = createLogger(LogPrefix.FORMATTING);
+const MAX_DIMENSION = 1500;
+const HEIC_MIME_TYPES = ['image/heic', 'image/heif', 'image/heic-sequence', 'image/heif-sequence'];
+let sharpModule = undefined;
+let heicConvertModule = undefined;
+async function tryLoadSharp() {
+    if (sharpModule !== undefined) {
+        return sharpModule;
+    }
+    try {
+        sharpModule = (await import('sharp')).default;
+        logger.log('sharp loaded successfully');
+        return sharpModule;
+    }
+    catch {
+        logger.log('sharp not available, images will be sent at original size');
+        sharpModule = null;
+        return null;
+    }
+}
+async function tryLoadHeicConvert() {
+    if (heicConvertModule !== undefined) {
+        return heicConvertModule;
+    }
+    try {
+        const mod = await import('heic-convert');
+        heicConvertModule = mod.default;
+        logger.log('heic-convert loaded successfully');
+        return heicConvertModule;
+    }
+    catch {
+        logger.log('heic-convert not available, HEIC images will be sent as-is');
+        heicConvertModule = null;
+        return null;
+    }
+}
+function isHeicMime(mime) {
+    return HEIC_MIME_TYPES.includes(mime.toLowerCase());
+}
+export async function processImage(buffer, mime) {
+    // Skip non-images (PDFs, etc.)
+    if (!mime.startsWith('image/')) {
+        return { buffer, mime };
+    }
+    let workingBuffer = buffer;
+    let workingMime = mime;
+    // Handle HEIC conversion first (before sharp, since sharp doesn't support HEIC)
+    if (isHeicMime(mime)) {
+        const heicConvert = await tryLoadHeicConvert();
+        if (heicConvert) {
+            try {
+                const outputArrayBuffer = await heicConvert({
+                    buffer: workingBuffer.buffer.slice(workingBuffer.byteOffset, workingBuffer.byteOffset + workingBuffer.byteLength),
+                    format: 'JPEG',
+                    quality: 0.85,
+                });
+                workingBuffer = Buffer.from(outputArrayBuffer);
+                workingMime = 'image/jpeg';
+                logger.log(`Converted HEIC to JPEG (${buffer.length} → ${workingBuffer.length} bytes)`);
+            }
+            catch (error) {
+                logger.error('Failed to convert HEIC, sending original:', error);
+                return { buffer, mime };
+            }
+        }
+        else {
+            // No heic-convert available, return original (LLM might not support it)
+            logger.log('HEIC image detected but heic-convert not available, sending as-is');
+            return { buffer, mime };
+        }
+    }
+    // Now process with sharp (resize + ensure JPEG output)
+    const sharp = await tryLoadSharp();
+    if (!sharp) {
+        return { buffer: workingBuffer, mime: workingMime };
+    }
+    try {
+        const image = sharp(workingBuffer);
+        const metadata = await image.metadata();
+        const { width, height } = metadata;
+        const needsResize = width && height && (width > MAX_DIMENSION || height > MAX_DIMENSION);
+        if (!needsResize) {
+            // Still convert to JPEG for consistency (unless already JPEG from HEIC conversion)
+            const outputBuffer = await image.jpeg({ quality: 85 }).toBuffer();
+            logger.log(`Converted image to JPEG: ${width}x${height} (${outputBuffer.length} bytes)`);
+            return { buffer: outputBuffer, mime: 'image/jpeg' };
+        }
+        // Resize and convert to JPEG
+        const outputBuffer = await image
+            .resize(MAX_DIMENSION, MAX_DIMENSION, {
+            fit: 'inside',
+            withoutEnlargement: true,
+        })
+            .jpeg({ quality: 85 })
+            .toBuffer();
+        logger.log(`Resized image: ${width}x${height} → max ${MAX_DIMENSION}px (${outputBuffer.length} bytes)`);
+        return { buffer: outputBuffer, mime: 'image/jpeg' };
+    }
+    catch (error) {
+        logger.error('Failed to process image with sharp, using working buffer:', error);
+        return { buffer: workingBuffer, mime: workingMime };
+    }
+}