@otto-assistant/otto 0.1.2 → 0.7.16

package/src/genai.ts

@@ -0,0 +1,321 @@
+// Google GenAI Live session manager for real-time voice interactions.
+// Establishes bidirectional audio streaming with Gemini, handles tool calls,
+// and manages the assistant's audio output for Discord voice channels.
+
+import {
+  GoogleGenAI,
+  LiveServerMessage,
+  MediaResolution,
+  Modality,
+  Session,
+} from '@google/genai'
+import type { CallableTool } from '@google/genai'
+import { writeFile } from 'fs'
+import type { AnyTool } from './ai-tool.js'
+
+import { createLogger, LogPrefix } from './logger.js'
+import { aiToolToCallableTool } from './ai-tool-to-genai.js'
+
+const genaiLogger = createLogger(LogPrefix.GENAI)
+
+const audioParts: Buffer[] = []
+
+function saveBinaryFile(fileName: string, content: Buffer) {
+  writeFile(fileName, content, 'utf8', (err) => {
+    if (err) {
+      genaiLogger.error(`Error writing file ${fileName}:`, err)
+      return
+    }
+    genaiLogger.log(`Appending stream content to file ${fileName}.`)
+  })
+}
+
+interface WavConversionOptions {
+  numChannels: number
+  sampleRate: number
+  bitsPerSample: number
+}
+
+function convertToWav(rawData: Buffer[], mimeType: string) {
+  const options = parseMimeType(mimeType)
+  const dataLength = rawData.reduce((a, b) => a + b.length, 0)
+  const wavHeader = createWavHeader(dataLength, options)
+  const buffer = Buffer.concat(rawData)
+
+  return Buffer.concat([wavHeader, buffer])
+}
+
+function parseMimeType(mimeType: string) {
+  const [fileType, ...params] = mimeType.split(';').map((s) => s.trim())
+  const [_, format] = fileType?.split('/') || []
+
+  const options: Partial<WavConversionOptions> = {
+    numChannels: 1,
+    bitsPerSample: 16,
+  }
+
+  if (format && format.startsWith('L')) {
+    const bits = parseInt(format.slice(1), 10)
+    if (!isNaN(bits)) {
+      options.bitsPerSample = bits
+    }
+  }
+
+  for (const param of params) {
+    const [key, value] = param.split('=').map((s) => s.trim())
+    if (key === 'rate') {
+      options.sampleRate = parseInt(value || '', 10)
+    }
+  }
+
+  return options as WavConversionOptions
+}
+
+function createWavHeader(dataLength: number, options: WavConversionOptions) {
+  const { numChannels, sampleRate, bitsPerSample } = options
+
+  // http://soundfile.sapp.org/doc/WaveFormat
+
+  const byteRate = (sampleRate * numChannels * bitsPerSample) / 8
+  const blockAlign = (numChannels * bitsPerSample) / 8
+  const buffer = Buffer.alloc(44)
+
+  buffer.write('RIFF', 0) // ChunkID
+  buffer.writeUInt32LE(36 + dataLength, 4) // ChunkSize
+  buffer.write('WAVE', 8) // Format
+  buffer.write('fmt ', 12) // Subchunk1ID
+  buffer.writeUInt32LE(16, 16) // Subchunk1Size (PCM)
+  buffer.writeUInt16LE(1, 20) // AudioFormat (1 = PCM)
+  buffer.writeUInt16LE(numChannels, 22) // NumChannels
+  buffer.writeUInt32LE(sampleRate, 24) // SampleRate
+  buffer.writeUInt32LE(byteRate, 28) // ByteRate
+  buffer.writeUInt16LE(blockAlign, 32) // BlockAlign
+  buffer.writeUInt16LE(bitsPerSample, 34) // BitsPerSample
+  buffer.write('data', 36) // Subchunk2ID
+  buffer.writeUInt32LE(dataLength, 40) // Subchunk2Size
+
+  return buffer
+}
+
+function defaultAudioChunkHandler({
+  data,
+  mimeType,
+}: {
+  data: Buffer
+  mimeType: string
+}) {
+  audioParts.push(data)
+  const fileName = 'audio.wav'
+  const buffer = convertToWav(audioParts, mimeType)
+  saveBinaryFile(fileName, buffer)
+}
+
+export async function startGenAiSession({
+  onAssistantAudioChunk,
+  onAssistantStartSpeaking,
+  onAssistantStopSpeaking,
+  onAssistantInterruptSpeaking,
+  systemMessage,
+  tools,
+  geminiApiKey,
+}: {
+  onAssistantAudioChunk?: (args: { data: Buffer; mimeType: string }) => void
+  onAssistantStartSpeaking?: () => void
+  onAssistantStopSpeaking?: () => void
+  onAssistantInterruptSpeaking?: () => void
+  systemMessage?: string
+  tools?: Record<string, AnyTool>
+  geminiApiKey?: string | null
+} = {}) {
+  let session: Session | undefined = undefined
+  const callableTools: Array<CallableTool & { name: string }> = []
+  let isAssistantSpeaking = false
+
+  const audioChunkHandler = onAssistantAudioChunk || defaultAudioChunkHandler
+
+  // Convert AI SDK tools to GenAI CallableTools
+  if (tools) {
+    for (const [name, tool] of Object.entries(tools)) {
+      callableTools.push(aiToolToCallableTool(tool, name))
+    }
+  }
+
+  function handleModelTurn(message: LiveServerMessage) {
+    if (message.toolCall) {
+      genaiLogger.log('Tool call:', message.toolCall)
+
+      // Handle tool calls
+      if (message.toolCall.functionCalls && callableTools.length > 0) {
+        for (const tool of callableTools) {
+          if (
+            !message.toolCall.functionCalls.some((x) => x.name === tool.name)
+          ) {
+            continue
+          }
+          tool
+            .callTool(message.toolCall.functionCalls)
+            .then((parts) => {
+              const functionResponses = parts
+                .filter((part) => part.functionResponse)
+                .map((part) => ({
+                  response: part.functionResponse!.response as Record<
+                    string,
+                    unknown
+                  >,
+                  id: part.functionResponse!.id,
+                  name: part.functionResponse!.name,
+                }))
+
+              if (functionResponses.length > 0 && session) {
+                session.sendToolResponse({ functionResponses })
+                genaiLogger.log(
+                  'client-toolResponse: ' +
+                    JSON.stringify({ functionResponses }),
+                )
+              }
+            })
+            .catch((error) => {
+              genaiLogger.error('Error handling tool calls:', error)
+            })
+        }
+      }
+    }
+    if (message.serverContent?.modelTurn?.parts) {
+      for (const part of message.serverContent.modelTurn.parts) {
+        if (part?.fileData) {
+          genaiLogger.log(`File: ${part?.fileData.fileUri}`)
+        }
+
+        if (part?.inlineData) {
+          const inlineData = part.inlineData
+          if (
+            !inlineData.mimeType ||
+            !inlineData.mimeType.startsWith('audio/')
+          ) {
+            genaiLogger.log(
+              'Skipping non-audio inlineData:',
+              inlineData.mimeType,
+            )
+            continue
+          }
+
+          // Trigger start speaking callback the first time audio is received
+          if (!isAssistantSpeaking && onAssistantStartSpeaking) {
+            isAssistantSpeaking = true
+            onAssistantStartSpeaking()
+          }
+
+          const buffer = Buffer.from(inlineData?.data ?? '', 'base64')
+          audioChunkHandler({
+            data: buffer,
+            mimeType: inlineData.mimeType ?? '',
+          })
+        }
+
+        if (part?.text) {
+          genaiLogger.log('Text:', part.text)
+        }
+      }
+    }
+    // Handle input transcription (user's audio transcription)
+    if (message.serverContent?.inputTranscription?.text) {
+      genaiLogger.log(
+        '[user transcription]',
+        message.serverContent.inputTranscription.text,
+      )
+    }
+
+    // Handle output transcription (model's audio transcription)
+    if (message.serverContent?.outputTranscription?.text) {
+      genaiLogger.log(
+        '[assistant transcription]',
+        message.serverContent.outputTranscription.text,
+      )
+    }
+    if (message.serverContent?.interrupted) {
+      genaiLogger.log('Assistant was interrupted')
+      if (isAssistantSpeaking && onAssistantInterruptSpeaking) {
+        isAssistantSpeaking = false
+        onAssistantInterruptSpeaking()
+      }
+    }
+    if (message.serverContent?.turnComplete) {
+      genaiLogger.log('Assistant turn complete')
+      if (isAssistantSpeaking && onAssistantStopSpeaking) {
+        isAssistantSpeaking = false
+        onAssistantStopSpeaking()
+      }
+    }
+  }
+
+  const apiKey = geminiApiKey || process.env.GEMINI_API_KEY
+
+  if (!apiKey) {
+    genaiLogger.error('No Gemini API key provided')
+    throw new Error('Gemini API key is required for voice interactions')
+  }
+
+  const ai = new GoogleGenAI({
+    apiKey,
+  })
+
+  const model = 'gemini-3.1-flash-live-preview'
+
+  session = await ai.live.connect({
+    model,
+    callbacks: {
+      onopen: function () {
+        genaiLogger.debug('Opened')
+      },
+      onmessage: function (message: LiveServerMessage) {
+        // genaiLogger.log(message)
+        try {
+          handleModelTurn(message)
+        } catch (error) {
+          genaiLogger.error('Error handling turn:', error)
+        }
+      },
+      onerror: function (e: { message?: string }) {
+        genaiLogger.debug('Error:', e.message)
+      },
+      onclose: function (e: { reason?: string }) {
+        genaiLogger.debug('Close:', e.reason)
+      },
+    },
+    config: {
+      tools: callableTools,
+      responseModalities: [Modality.AUDIO],
+      mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+      inputAudioTranscription: {}, // transcribes your input speech
+      outputAudioTranscription: {}, // transcribes the model's spoken audio
+      systemInstruction: {
+        parts: [
+          {
+            text: systemMessage || '',
+          },
+        ],
+      },
+      speechConfig: {
+        voiceConfig: {
+          prebuiltVoiceConfig: {
+            voiceName: 'Charon', // Orus also not bad
+          },
+        },
+      },
+      contextWindowCompression: {
+        triggerTokens: '25600',
+
+        slidingWindow: { targetTokens: '12800' },
+      },
+    },
+  })
+
+  return {
+    session,
+    stop: () => {
+      const currentSession = session
+      session = undefined
+      currentSession?.close()
+    },
+  }
+}

package/src/generated/browser.ts

@@ -0,0 +1,114 @@
+
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// biome-ignore-all lint: generated file
+// @ts-nocheck 
+/*
+ * This file should be your main import to use Prisma-related types and utilities in a browser. 
+ * Use it to get access to models, enums, and input types.
+ * 
+ * This file does not contain a `PrismaClient` class, nor several other helpers that are intended as server-side only.
+ * See `client.ts` for the standard, server-side entry point.
+ *
+ * 🟢 You can import this file directly.
+ */
+
+import * as Prisma from './internal/prismaNamespaceBrowser.js'
+export { Prisma }
+export * as $Enums from './enums.js'
+export * from './enums.js';
+/**
+ * Model thread_sessions
+ * 
+ */
+export type thread_sessions = Prisma.thread_sessionsModel
+/**
+ * Model session_events
+ * 
+ */
+export type session_events = Prisma.session_eventsModel
+/**
+ * Model part_messages
+ * 
+ */
+export type part_messages = Prisma.part_messagesModel
+/**
+ * Model bot_tokens
+ * 
+ */
+export type bot_tokens = Prisma.bot_tokensModel
+/**
+ * Model channel_directories
+ * 
+ */
+export type channel_directories = Prisma.channel_directoriesModel
+/**
+ * Model bot_api_keys
+ * 
+ */
+export type bot_api_keys = Prisma.bot_api_keysModel
+/**
+ * Model thread_worktrees
+ * 
+ */
+export type thread_worktrees = Prisma.thread_worktreesModel
+/**
+ * Model channel_models
+ * 
+ */
+export type channel_models = Prisma.channel_modelsModel
+/**
+ * Model session_models
+ * 
+ */
+export type session_models = Prisma.session_modelsModel
+/**
+ * Model channel_agents
+ * 
+ */
+export type channel_agents = Prisma.channel_agentsModel
+/**
+ * Model session_agents
+ * 
+ */
+export type session_agents = Prisma.session_agentsModel
+/**
+ * Model channel_worktrees
+ * 
+ */
+export type channel_worktrees = Prisma.channel_worktreesModel
+/**
+ * Model channel_verbosity
+ * 
+ */
+export type channel_verbosity = Prisma.channel_verbosityModel
+/**
+ * Model channel_mention_mode
+ * 
+ */
+export type channel_mention_mode = Prisma.channel_mention_modeModel
+/**
+ * Model global_models
+ * 
+ */
+export type global_models = Prisma.global_modelsModel
+/**
+ * Model scheduled_tasks
+ * 
+ */
+export type scheduled_tasks = Prisma.scheduled_tasksModel
+/**
+ * Model session_start_sources
+ * 
+ */
+export type session_start_sources = Prisma.session_start_sourcesModel
+/**
+ * Model forum_sync_configs
+ * 
+ */
+export type forum_sync_configs = Prisma.forum_sync_configsModel
+/**
+ * Model ipc_requests
+ * 
+ */
+export type ipc_requests = Prisma.ipc_requestsModel

package/src/generated/client.ts

@@ -0,0 +1,138 @@
+
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// biome-ignore-all lint: generated file
+// @ts-nocheck 
+/*
+ * This file should be your main import to use Prisma. Through it you get access to all the models, enums, and input types.
+ * If you're looking for something you can import in the client-side of your application, please refer to the `browser.ts` file instead.
+ *
+ * 🟢 You can import this file directly.
+ */
+
+import * as process from 'node:process'
+import * as path from 'node:path'
+import { fileURLToPath } from 'node:url'
+globalThis['__dirname'] = path.dirname(fileURLToPath(import.meta.url))
+
+import * as runtime from "@prisma/client/runtime/client"
+import * as $Enums from "./enums.js"
+import * as $Class from "./internal/class.js"
+import * as Prisma from "./internal/prismaNamespace.js"
+
+export * as $Enums from './enums.js'
+export * from "./enums.js"
+/**
+ * ## Prisma Client
+ * 
+ * Type-safe database client for TypeScript
+ * @example
+ * ```
+ * const prisma = new PrismaClient({
+ *   adapter: new PrismaPg({ connectionString: process.env.DATABASE_URL })
+ * })
+ * // Fetch zero or more Thread_sessions
+ * const thread_sessions = await prisma.thread_sessions.findMany()
+ * ```
+ * 
+ * Read more in our [docs](https://pris.ly/d/client).
+ */
+export const PrismaClient = $Class.getPrismaClientClass()
+export type PrismaClient<LogOpts extends Prisma.LogLevel = never, OmitOpts extends Prisma.PrismaClientOptions["omit"] = Prisma.PrismaClientOptions["omit"], ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = $Class.PrismaClient<LogOpts, OmitOpts, ExtArgs>
+export { Prisma }
+
+/**
+ * Model thread_sessions
+ * 
+ */
+export type thread_sessions = Prisma.thread_sessionsModel
+/**
+ * Model session_events
+ * 
+ */
+export type session_events = Prisma.session_eventsModel
+/**
+ * Model part_messages
+ * 
+ */
+export type part_messages = Prisma.part_messagesModel
+/**
+ * Model bot_tokens
+ * 
+ */
+export type bot_tokens = Prisma.bot_tokensModel
+/**
+ * Model channel_directories
+ * 
+ */
+export type channel_directories = Prisma.channel_directoriesModel
+/**
+ * Model bot_api_keys
+ * 
+ */
+export type bot_api_keys = Prisma.bot_api_keysModel
+/**
+ * Model thread_worktrees
+ * 
+ */
+export type thread_worktrees = Prisma.thread_worktreesModel
+/**
+ * Model channel_models
+ * 
+ */
+export type channel_models = Prisma.channel_modelsModel
+/**
+ * Model session_models
+ * 
+ */
+export type session_models = Prisma.session_modelsModel
+/**
+ * Model channel_agents
+ * 
+ */
+export type channel_agents = Prisma.channel_agentsModel
+/**
+ * Model session_agents
+ * 
+ */
+export type session_agents = Prisma.session_agentsModel
+/**
+ * Model channel_worktrees
+ * 
+ */
+export type channel_worktrees = Prisma.channel_worktreesModel
+/**
+ * Model channel_verbosity
+ * 
+ */
+export type channel_verbosity = Prisma.channel_verbosityModel
+/**
+ * Model channel_mention_mode
+ * 
+ */
+export type channel_mention_mode = Prisma.channel_mention_modeModel
+/**
+ * Model global_models
+ * 
+ */
+export type global_models = Prisma.global_modelsModel
+/**
+ * Model scheduled_tasks
+ * 
+ */
+export type scheduled_tasks = Prisma.scheduled_tasksModel
+/**
+ * Model session_start_sources
+ * 
+ */
+export type session_start_sources = Prisma.session_start_sourcesModel
+/**
+ * Model forum_sync_configs
+ * 
+ */
+export type forum_sync_configs = Prisma.forum_sync_configsModel
+/**
+ * Model ipc_requests
+ * 
+ */
+export type ipc_requests = Prisma.ipc_requestsModel