@brianli/kimaki 0.4.72-brianli.1

package/src/genai-worker.ts

@@ -0,0 +1,386 @@
+// Worker thread for GenAI voice processing.
+// Runs in a separate thread to handle audio encoding/decoding without blocking.
+// Resamples 24kHz GenAI output to 48kHz stereo Opus packets for Discord.
+
+import { parentPort, threadId } from 'node:worker_threads'
+import { createWriteStream, type WriteStream } from 'node:fs'
+import path from 'node:path'
+import * as errore from 'errore'
+import { Resampler } from '@purinton/resampler'
+import * as prism from 'prism-media'
+import { startGenAiSession } from './genai.js'
+import type { Session } from '@google/genai'
+import { getTools } from './tools.js'
+import { mkdir } from 'node:fs/promises'
+import type { WorkerInMessage, WorkerOutMessage } from './worker-types.js'
+import { createLogger, formatErrorWithStack, LogPrefix } from './logger.js'
+import { initSentry, notifyError } from './sentry.js'
+
+if (!parentPort) {
+  throw new Error('This module must be run as a worker thread')
+}
+
+const workerLogger = createLogger(`${LogPrefix.WORKER}_${threadId}`)
+workerLogger.log('GenAI worker started')
+
+// Initialize Sentry in worker thread (inherits KIMAKI_SENTRY_DSN from parent)
+initSentry()
+
+// Define sendError early so it can be used by global handlers
+function sendError(error: string) {
+  if (parentPort) {
+    parentPort.postMessage({
+      type: 'error',
+      error,
+    } satisfies WorkerOutMessage)
+  }
+}
+
+// Add global error handlers for the worker thread
+process.on('uncaughtException', (error) => {
+  workerLogger.error('Uncaught exception in worker:', error)
+  void notifyError(error, 'Uncaught exception in GenAI worker')
+  sendError(`Worker crashed: ${error.message}`)
+  process.exit(1)
+})
+
+process.on('unhandledRejection', (reason, promise) => {
+  const formattedReason = formatErrorWithStack(reason)
+  workerLogger.error(
+    'Unhandled rejection in worker:',
+    formattedReason,
+    'at promise:',
+    promise,
+  )
+  const error =
+    reason instanceof Error
+      ? reason
+      : new Error(formattedReason)
+  void notifyError(error, 'Unhandled rejection in GenAI worker')
+  sendError(`Worker unhandled rejection: ${formattedReason}`)
+})
+
+// Audio configuration
+const AUDIO_CONFIG = {
+  inputSampleRate: 24000, // GenAI output
+  inputChannels: 1,
+  outputSampleRate: 48000, // Discord expects
+  outputChannels: 2,
+  opusFrameSize: 960, // 20ms at 48kHz
+}
+
+// Initialize audio processing components
+const resampler = new Resampler({
+  inRate: AUDIO_CONFIG.inputSampleRate,
+  outRate: AUDIO_CONFIG.outputSampleRate,
+  inChannels: AUDIO_CONFIG.inputChannels,
+  outChannels: AUDIO_CONFIG.outputChannels,
+  volume: 1,
+  filterWindow: 8,
+})
+
+const opusEncoder = new prism.opus.Encoder({
+  rate: AUDIO_CONFIG.outputSampleRate,
+  channels: AUDIO_CONFIG.outputChannels,
+  frameSize: AUDIO_CONFIG.opusFrameSize,
+})
+
+// Pipe resampler to encoder with error handling
+resampler.pipe(opusEncoder).on('error', (error) => {
+  workerLogger.error('Pipe error between resampler and encoder:', error)
+  void notifyError(error, 'GenAI worker audio pipeline error')
+  sendError(`Audio pipeline error: ${error.message}`)
+})
+
+// Opus packet queue and interval for 20ms packet sending
+const opusPacketQueue: Buffer[] = []
+let packetInterval: NodeJS.Timeout | null = null
+
+// Send packets every 20ms
+function startPacketSending() {
+  if (packetInterval) return
+
+  packetInterval = setInterval(() => {
+    const packet = opusPacketQueue.shift()
+    if (!packet) return
+
+    // Transfer packet as ArrayBuffer
+    const arrayBuffer = packet.buffer.slice(
+      packet.byteOffset,
+      packet.byteOffset + packet.byteLength,
+    ) as ArrayBuffer
+
+    parentPort!.postMessage(
+      {
+        type: 'assistantOpusPacket',
+        packet: arrayBuffer,
+      } satisfies WorkerOutMessage,
+      [arrayBuffer], // Transfer ownership
+    )
+  }, 20)
+}
+
+function stopPacketSending() {
+  if (packetInterval) {
+    clearInterval(packetInterval)
+    packetInterval = null
+  }
+  opusPacketQueue.length = 0
+}
+
+// Session state
+let session: { session: Session; stop: () => void } | null = null
+
+// Audio log stream for assistant audio
+let audioLogStream: WriteStream | null = null
+
+// Create assistant audio log stream for debugging
+async function createAssistantAudioLogStream(
+  guildId: string,
+  channelId: string,
+): Promise<WriteStream | null> {
+  if (!process.env.DEBUG) return null
+
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-')
+  const audioDir = path.join(
+    process.cwd(),
+    'discord-audio-logs',
+    guildId,
+    channelId,
+  )
+
+  const mkdirError = await errore.tryAsync({
+    try: () => mkdir(audioDir, { recursive: true }),
+    catch: (e) => e as Error,
+  })
+  if (mkdirError instanceof Error) {
+    workerLogger.error(
+      `Failed to create audio log directory:`,
+      mkdirError.message,
+    )
+    return null
+  }
+
+  // Create stream for assistant audio (24kHz mono s16le PCM)
+  const outputFileName = `assistant_${timestamp}.24.pcm`
+  const outputFilePath = path.join(audioDir, outputFileName)
+  const outputAudioStream = createWriteStream(outputFilePath)
+
+  // Add error handler to prevent crashes
+  outputAudioStream.on('error', (error) => {
+    workerLogger.error(`Assistant audio log stream error:`, error)
+  })
+
+  workerLogger.log(`Created assistant audio log: ${outputFilePath}`)
+
+  return outputAudioStream
+}
+
+// Handle encoded Opus packets
+opusEncoder.on('data', (packet: Buffer) => {
+  opusPacketQueue.push(packet)
+})
+
+// Handle stream end events
+opusEncoder.on('end', () => {
+  workerLogger.log('Opus encoder stream ended')
+})
+
+resampler.on('end', () => {
+  workerLogger.log('Resampler stream ended')
+})
+
+// Handle errors
+resampler.on('error', (error: unknown) => {
+  workerLogger.error(`Resampler error:`, error)
+  void notifyError(error, 'GenAI worker resampler error')
+  sendError(`Resampler error: ${(error as Error).message}`)
+})
+
+opusEncoder.on('error', (error: unknown) => {
+  workerLogger.error(`Encoder error:`, error)
+  const errMsg = (error as Error).message || ''
+  // Check for specific corrupted data errors
+  if (errMsg.includes('The compressed data passed is corrupted')) {
+    workerLogger.warn('Received corrupted audio data in opus encoder')
+  } else {
+    void notifyError(error, 'GenAI worker encoder error')
+    sendError(`Encoder error: ${errMsg}`)
+  }
+})
+
+async function cleanupAsync(): Promise<void> {
+  workerLogger.log(`Starting async cleanup`)
+
+  stopPacketSending()
+
+  if (session) {
+    workerLogger.log(`Stopping GenAI session`)
+    session.stop()
+    session = null
+  }
+
+  // Wait for audio log stream to finish writing
+  if (audioLogStream) {
+    workerLogger.log(`Closing assistant audio log stream`)
+    await new Promise<void>((resolve, reject) => {
+      audioLogStream!.end(() => {
+        workerLogger.log(`Assistant audio log stream closed`)
+        resolve()
+      })
+      audioLogStream!.on('error', reject)
+      // Add timeout to prevent hanging
+      setTimeout(() => {
+        workerLogger.log(`Audio stream close timeout, continuing`)
+        resolve()
+      }, 3000)
+    })
+    audioLogStream = null
+  }
+
+  // Unpipe and end the encoder first
+  resampler.unpipe(opusEncoder)
+
+  // End the encoder stream
+  await new Promise<void>((resolve) => {
+    opusEncoder.end(() => {
+      workerLogger.log(`Opus encoder ended`)
+      resolve()
+    })
+    // Add timeout
+    setTimeout(resolve, 1000)
+  })
+
+  // End the resampler stream
+  await new Promise<void>((resolve) => {
+    resampler.end(() => {
+      workerLogger.log(`Resampler ended`)
+      resolve()
+    })
+    // Add timeout
+    setTimeout(resolve, 1000)
+  })
+
+  workerLogger.log(`Async cleanup complete`)
+}
+
+// Handle messages from main thread
+parentPort.on('message', async (message: WorkerInMessage) => {
+  try {
+    switch (message.type) {
+      case 'init': {
+        workerLogger.log(`Initializing with directory:`, message.directory)
+
+        // Create audio log stream for assistant audio
+        audioLogStream = await createAssistantAudioLogStream(
+          message.guildId,
+          message.channelId,
+        )
+
+        // Start packet sending interval
+        startPacketSending()
+
+        // Get tools for the directory
+        const { tools } = await getTools({
+          directory: message.directory,
+          onMessageCompleted: (params) => {
+            parentPort!.postMessage({
+              type: 'toolCallCompleted',
+              ...params,
+            } satisfies WorkerOutMessage)
+          },
+        })
+
+        // Start GenAI session
+        session = await startGenAiSession({
+          tools,
+          systemMessage: message.systemMessage,
+          geminiApiKey: message.geminiApiKey,
+          onAssistantAudioChunk({ data }) {
+            // Write to audio log if enabled
+            if (audioLogStream && !audioLogStream.destroyed) {
+              audioLogStream.write(data, (err) => {
+                if (err) {
+                  workerLogger.error('Error writing to audio log:', err)
+                }
+              })
+            }
+
+            // Write PCM data to resampler which will output Opus packets
+            if (!resampler.destroyed) {
+              resampler.write(data, (err) => {
+                if (err) {
+                  workerLogger.error('Error writing to resampler:', err)
+                  sendError(`Failed to process audio: ${err.message}`)
+                }
+              })
+            }
+          },
+          onAssistantStartSpeaking() {
+            parentPort!.postMessage({
+              type: 'assistantStartSpeaking',
+            } satisfies WorkerOutMessage)
+          },
+          onAssistantStopSpeaking() {
+            parentPort!.postMessage({
+              type: 'assistantStopSpeaking',
+            } satisfies WorkerOutMessage)
+          },
+          onAssistantInterruptSpeaking() {
+            parentPort!.postMessage({
+              type: 'assistantInterruptSpeaking',
+            } satisfies WorkerOutMessage)
+          },
+        })
+
+        // Notify main thread we're ready
+        parentPort!.postMessage({
+          type: 'ready',
+        } satisfies WorkerOutMessage)
+        break
+      }
+
+      case 'sendRealtimeInput': {
+        if (!session) {
+          sendError('Session not initialized')
+          return
+        }
+        session.session.sendRealtimeInput({
+          audio: message.audio,
+          audioStreamEnd: message.audioStreamEnd,
+        })
+        break
+      }
+
+      case 'sendTextInput': {
+        if (!session) {
+          sendError('Session not initialized')
+          return
+        }
+        session.session.sendRealtimeInput({
+          text: message.text,
+        })
+        break
+      }
+
+      case 'interrupt': {
+        workerLogger.log(`Interrupting playback`)
+        // Clear the opus packet queue
+        opusPacketQueue.length = 0
+        break
+      }
+
+      case 'stop': {
+        workerLogger.log(`Stopping worker`)
+        await cleanupAsync()
+        // process.exit(0)
+        break
+      }
+    }
+  } catch (error) {
+    workerLogger.error(`Error handling message:`, error)
+    sendError(
+      error instanceof Error ? error.message : 'Unknown error in worker',
+    )
+  }
+})

package/src/genai.ts

@@ -0,0 +1,321 @@
+// Google GenAI Live session manager for real-time voice interactions.
+// Establishes bidirectional audio streaming with Gemini, handles tool calls,
+// and manages the assistant's audio output for Discord voice channels.
+
+import {
+  GoogleGenAI,
+  LiveServerMessage,
+  MediaResolution,
+  Modality,
+  Session,
+} from '@google/genai'
+import type { CallableTool } from '@google/genai'
+import { writeFile } from 'fs'
+import type { AnyTool } from './ai-tool.js'
+
+import { createLogger, LogPrefix } from './logger.js'
+import { aiToolToCallableTool } from './ai-tool-to-genai.js'
+
+const genaiLogger = createLogger(LogPrefix.GENAI)
+
+const audioParts: Buffer[] = []
+
+function saveBinaryFile(fileName: string, content: Buffer) {
+  writeFile(fileName, content, 'utf8', (err) => {
+    if (err) {
+      genaiLogger.error(`Error writing file ${fileName}:`, err)
+      return
+    }
+    genaiLogger.log(`Appending stream content to file ${fileName}.`)
+  })
+}
+
+interface WavConversionOptions {
+  numChannels: number
+  sampleRate: number
+  bitsPerSample: number
+}
+
+function convertToWav(rawData: Buffer[], mimeType: string) {
+  const options = parseMimeType(mimeType)
+  const dataLength = rawData.reduce((a, b) => a + b.length, 0)
+  const wavHeader = createWavHeader(dataLength, options)
+  const buffer = Buffer.concat(rawData)
+
+  return Buffer.concat([wavHeader, buffer])
+}
+
+function parseMimeType(mimeType: string) {
+  const [fileType, ...params] = mimeType.split(';').map((s) => s.trim())
+  const [_, format] = fileType?.split('/') || []
+
+  const options: Partial<WavConversionOptions> = {
+    numChannels: 1,
+    bitsPerSample: 16,
+  }
+
+  if (format && format.startsWith('L')) {
+    const bits = parseInt(format.slice(1), 10)
+    if (!isNaN(bits)) {
+      options.bitsPerSample = bits
+    }
+  }
+
+  for (const param of params) {
+    const [key, value] = param.split('=').map((s) => s.trim())
+    if (key === 'rate') {
+      options.sampleRate = parseInt(value || '', 10)
+    }
+  }
+
+  return options as WavConversionOptions
+}
+
+function createWavHeader(dataLength: number, options: WavConversionOptions) {
+  const { numChannels, sampleRate, bitsPerSample } = options
+
+  // http://soundfile.sapp.org/doc/WaveFormat
+
+  const byteRate = (sampleRate * numChannels * bitsPerSample) / 8
+  const blockAlign = (numChannels * bitsPerSample) / 8
+  const buffer = Buffer.alloc(44)
+
+  buffer.write('RIFF', 0) // ChunkID
+  buffer.writeUInt32LE(36 + dataLength, 4) // ChunkSize
+  buffer.write('WAVE', 8) // Format
+  buffer.write('fmt ', 12) // Subchunk1ID
+  buffer.writeUInt32LE(16, 16) // Subchunk1Size (PCM)
+  buffer.writeUInt16LE(1, 20) // AudioFormat (1 = PCM)
+  buffer.writeUInt16LE(numChannels, 22) // NumChannels
+  buffer.writeUInt32LE(sampleRate, 24) // SampleRate
+  buffer.writeUInt32LE(byteRate, 28) // ByteRate
+  buffer.writeUInt16LE(blockAlign, 32) // BlockAlign
+  buffer.writeUInt16LE(bitsPerSample, 34) // BitsPerSample
+  buffer.write('data', 36) // Subchunk2ID
+  buffer.writeUInt32LE(dataLength, 40) // Subchunk2Size
+
+  return buffer
+}
+
+function defaultAudioChunkHandler({
+  data,
+  mimeType,
+}: {
+  data: Buffer
+  mimeType: string
+}) {
+  audioParts.push(data)
+  const fileName = 'audio.wav'
+  const buffer = convertToWav(audioParts, mimeType)
+  saveBinaryFile(fileName, buffer)
+}
+
+export async function startGenAiSession({
+  onAssistantAudioChunk,
+  onAssistantStartSpeaking,
+  onAssistantStopSpeaking,
+  onAssistantInterruptSpeaking,
+  systemMessage,
+  tools,
+  geminiApiKey,
+}: {
+  onAssistantAudioChunk?: (args: { data: Buffer; mimeType: string }) => void
+  onAssistantStartSpeaking?: () => void
+  onAssistantStopSpeaking?: () => void
+  onAssistantInterruptSpeaking?: () => void
+  systemMessage?: string
+  tools?: Record<string, AnyTool>
+  geminiApiKey?: string | null
+} = {}) {
+  let session: Session | undefined = undefined
+  const callableTools: Array<CallableTool & { name: string }> = []
+  let isAssistantSpeaking = false
+
+  const audioChunkHandler = onAssistantAudioChunk || defaultAudioChunkHandler
+
+  // Convert AI SDK tools to GenAI CallableTools
+  if (tools) {
+    for (const [name, tool] of Object.entries(tools)) {
+      callableTools.push(aiToolToCallableTool(tool, name))
+    }
+  }
+
+  function handleModelTurn(message: LiveServerMessage) {
+    if (message.toolCall) {
+      genaiLogger.log('Tool call:', message.toolCall)
+
+      // Handle tool calls
+      if (message.toolCall.functionCalls && callableTools.length > 0) {
+        for (const tool of callableTools) {
+          if (
+            !message.toolCall.functionCalls.some((x) => x.name === tool.name)
+          ) {
+            continue
+          }
+          tool
+            .callTool(message.toolCall.functionCalls)
+            .then((parts) => {
+              const functionResponses = parts
+                .filter((part) => part.functionResponse)
+                .map((part) => ({
+                  response: part.functionResponse!.response as Record<
+                    string,
+                    unknown
+                  >,
+                  id: part.functionResponse!.id,
+                  name: part.functionResponse!.name,
+                }))
+
+              if (functionResponses.length > 0 && session) {
+                session.sendToolResponse({ functionResponses })
+                genaiLogger.log(
+                  'client-toolResponse: ' +
+                    JSON.stringify({ functionResponses }),
+                )
+              }
+            })
+            .catch((error) => {
+              genaiLogger.error('Error handling tool calls:', error)
+            })
+        }
+      }
+    }
+    if (message.serverContent?.modelTurn?.parts) {
+      for (const part of message.serverContent.modelTurn.parts) {
+        if (part?.fileData) {
+          genaiLogger.log(`File: ${part?.fileData.fileUri}`)
+        }
+
+        if (part?.inlineData) {
+          const inlineData = part.inlineData
+          if (
+            !inlineData.mimeType ||
+            !inlineData.mimeType.startsWith('audio/')
+          ) {
+            genaiLogger.log(
+              'Skipping non-audio inlineData:',
+              inlineData.mimeType,
+            )
+            continue
+          }
+
+          // Trigger start speaking callback the first time audio is received
+          if (!isAssistantSpeaking && onAssistantStartSpeaking) {
+            isAssistantSpeaking = true
+            onAssistantStartSpeaking()
+          }
+
+          const buffer = Buffer.from(inlineData?.data ?? '', 'base64')
+          audioChunkHandler({
+            data: buffer,
+            mimeType: inlineData.mimeType ?? '',
+          })
+        }
+
+        if (part?.text) {
+          genaiLogger.log('Text:', part.text)
+        }
+      }
+    }
+    // Handle input transcription (user's audio transcription)
+    if (message.serverContent?.inputTranscription?.text) {
+      genaiLogger.log(
+        '[user transcription]',
+        message.serverContent.inputTranscription.text,
+      )
+    }
+
+    // Handle output transcription (model's audio transcription)
+    if (message.serverContent?.outputTranscription?.text) {
+      genaiLogger.log(
+        '[assistant transcription]',
+        message.serverContent.outputTranscription.text,
+      )
+    }
+    if (message.serverContent?.interrupted) {
+      genaiLogger.log('Assistant was interrupted')
+      if (isAssistantSpeaking && onAssistantInterruptSpeaking) {
+        isAssistantSpeaking = false
+        onAssistantInterruptSpeaking()
+      }
+    }
+    if (message.serverContent?.turnComplete) {
+      genaiLogger.log('Assistant turn complete')
+      if (isAssistantSpeaking && onAssistantStopSpeaking) {
+        isAssistantSpeaking = false
+        onAssistantStopSpeaking()
+      }
+    }
+  }
+
+  const apiKey = geminiApiKey || process.env.GEMINI_API_KEY
+
+  if (!apiKey) {
+    genaiLogger.error('No Gemini API key provided')
+    throw new Error('Gemini API key is required for voice interactions')
+  }
+
+  const ai = new GoogleGenAI({
+    apiKey,
+  })
+
+  const model = 'gemini-2.5-flash-native-audio-preview-12-2025'
+
+  session = await ai.live.connect({
+    model,
+    callbacks: {
+      onopen: function () {
+        genaiLogger.debug('Opened')
+      },
+      onmessage: function (message: LiveServerMessage) {
+        // genaiLogger.log(message)
+        try {
+          handleModelTurn(message)
+        } catch (error) {
+          genaiLogger.error('Error handling turn:', error)
+        }
+      },
+      onerror: function (e: ErrorEvent) {
+        genaiLogger.debug('Error:', e.message)
+      },
+      onclose: function (e: CloseEvent) {
+        genaiLogger.debug('Close:', e.reason)
+      },
+    },
+    config: {
+      tools: callableTools,
+      responseModalities: [Modality.AUDIO],
+      mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+      inputAudioTranscription: {}, // transcribes your input speech
+      outputAudioTranscription: {}, // transcribes the model's spoken audio
+      systemInstruction: {
+        parts: [
+          {
+            text: systemMessage || '',
+          },
+        ],
+      },
+      speechConfig: {
+        voiceConfig: {
+          prebuiltVoiceConfig: {
+            voiceName: 'Charon', // Orus also not bad
+          },
+        },
+      },
+      contextWindowCompression: {
+        triggerTokens: '25600',
+
+        slidingWindow: { targetTokens: '12800' },
+      },
+    },
+  })
+
+  return {
+    session,
+    stop: () => {
+      const currentSession = session
+      session = undefined
+      currentSession?.close()
+    },
+  }
+}