@mastra/voice-google-gemini-live 0.0.0-add-libsql-changeset-20250910154739

package/dist/index.d.ts

@@ -0,0 +1,436 @@
+import type { ToolsInput } from '@mastra/core/agent';
+import { MastraVoice } from '@mastra/core/voice';
+import type { VoiceEventType, VoiceConfig } from '@mastra/core/voice';
+import type { GeminiLiveVoiceConfig, GeminiLiveVoiceOptions, GeminiLiveEventMap, GeminiSessionConfig } from './types';
+/**
+ * Helper class for consistent error handling
+ */
+/**
+ * GeminiLiveVoice provides real-time multimodal voice interactions using Google's Gemini Live API.
+ *
+ * Features:
+ * - Bidirectional audio streaming
+ * - Built-in VAD and interrupt handling
+ * - Tool calling capabilities
+ * - Session management and resumption
+ * - Live transcription
+ * - Support for both Gemini API and Vertex AI
+ *
+ * @example Backward compatibility - Direct options (legacy)
+ * ```typescript
+ * const voice = new GeminiLiveVoice({
+ *   apiKey: 'your-api-key',
+ *   model: 'gemini-2.0-flash-live-001',
+ *   speaker: 'Puck',
+ *   instructions: 'You are a helpful assistant'
+ * });
+ * ```
+ *
+ * @example Mastra VoiceConfig pattern - Recommended
+ * ```typescript
+ * const voice = new GeminiLiveVoice({
+ *   speechModel: { name: 'gemini-2.0-flash-live-001', apiKey: 'your-api-key' },
+ *   speaker: 'Puck',
+ *   realtimeConfig: {
+ *     model: 'gemini-2.0-flash-live-001',
+ *     apiKey: 'your-api-key',
+ *     options: {
+ *       instructions: 'You are a helpful assistant',
+ *       debug: true
+ *     }
+ *   }
+ * });
+ * ```
+ *
+ * @example Using Vertex AI (with OAuth)
+ * ```typescript
+ * const voice = new GeminiLiveVoice({
+ *   realtimeConfig: {
+ *     model: 'gemini-2.0-flash-live-001',
+ *     options: {
+ *       vertexAI: true,
+ *       project: 'your-gcp-project',
+ *       location: 'us-central1',
+ *       serviceAccountKeyFile: '/path/to/service-account.json',
+ *     }
+ *   }
+ * });
+ * ```
+ */
+export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig, GeminiLiveVoiceOptions, GeminiLiveVoiceOptions, ToolsInput, GeminiLiveEventMap> {
+    private ws?;
+    private eventManager;
+    private state;
+    private sessionHandle?;
+    private readonly debug;
+    private readonly audioConfig;
+    private queue;
+    private connectionManager;
+    private contextManager;
+    private authManager;
+    private audioStreamManager;
+    private sessionId?;
+    private sessionStartTime?;
+    private isResuming;
+    private sessionDurationTimeout?;
+    private tools?;
+    private runtimeContext?;
+    private options;
+    /**
+     * Normalize configuration to ensure proper VoiceConfig format
+     * Handles backward compatibility with direct GeminiLiveVoiceConfig
+     * @private
+     */
+    private static normalizeConfig;
+    /**
+     * Creates a new GeminiLiveVoice instance
+     *
+     * @param config Configuration options
+     */
+    constructor(config?: VoiceConfig<GeminiLiveVoiceConfig> | GeminiLiveVoiceConfig);
+    /**
+     * Register an event listener
+     * @param event Event name (e.g., 'speaking', 'writing', 'error', 'speaker')
+     * @param callback Callback function that receives event data
+     *
+     * @example
+     * ```typescript
+     * // Listen for audio responses
+     * voice.on('speaking', ({ audio, audioData, sampleRate }) => {
+     *   console.log('Received audio chunk:', audioData.length);
+     * });
+     *
+     * // Listen for text responses and transcriptions
+     * voice.on('writing', ({ text, role }) => {
+     *   console.log(`${role}: ${text}`);
+     * });
+     *
+     * // Listen for audio streams (for concatenated playback)
+     * voice.on('speaker', (audioStream) => {
+     *   audioStream.pipe(playbackDevice);
+     * });
+     *
+     * // Handle errors
+     * voice.on('error', ({ message, code, details }) => {
+     *   console.error('Voice error:', message);
+     * });
+     * ```
+     */
+    on<E extends VoiceEventType>(event: E, callback: (data: E extends keyof GeminiLiveEventMap ? GeminiLiveEventMap[E] : unknown) => void): void;
+    /**
+     * Remove an event listener
+     * @param event Event name
+     * @param callback Callback function to remove
+     */
+    off<E extends VoiceEventType>(event: E, callback: (data: E extends keyof GeminiLiveEventMap ? GeminiLiveEventMap[E] : unknown) => void): void;
+    /**
+     * Register a one-time event listener that automatically removes itself after the first emission
+     * @param event Event name
+     * @param callback Callback function that receives event data
+     */
+    once<E extends VoiceEventType>(event: E, callback: (data: E extends keyof GeminiLiveEventMap ? GeminiLiveEventMap[E] : unknown) => void): void;
+    /**
+     * Emit an event to listeners with improved error handling
+     * @private
+     */
+    private emit;
+    /**
+     * Clean up event listeners to prevent memory leaks
+     * @private
+     */
+    private cleanupEventListeners;
+    /**
+     * Get current event listener information for debugging
+     * @returns Object with event names and listener counts
+     */
+    getEventListenerInfo(): Record<string, number>;
+    /**
+     * Create and emit a standardized error
+     * @private
+     */
+    private createAndEmitError;
+    /**
+     * Handle connection state validation with standardized errors
+     * @private
+     */
+    private validateConnectionState;
+    /**
+     * Handle WebSocket state validation with standardized errors
+     * @private
+     */
+    private validateWebSocketState;
+    /**
+     * Establish connection to the Gemini Live API
+     */
+    connect({ runtimeContext }?: {
+        runtimeContext?: any;
+    }): Promise<void>;
+    /**
+     * Disconnect from the Gemini Live API
+     */
+    disconnect(): Promise<void>;
+    /**
+     * Send text to be converted to speech
+     */
+    speak(input: string | NodeJS.ReadableStream, options?: GeminiLiveVoiceOptions): Promise<void>;
+    /**
+     * Send audio stream for processing
+     */
+    send(audioData: NodeJS.ReadableStream | Int16Array): Promise<void>;
+    /**
+     * Process speech from audio stream (traditional STT interface)
+     */
+    listen(audioStream: NodeJS.ReadableStream, _options?: GeminiLiveVoiceOptions): Promise<string>;
+    /**
+     * Get available speakers/voices
+     */
+    getSpeakers(): Promise<Array<{
+        voiceId: string;
+        description?: string;
+    }>>;
+    /**
+     * Resume a previous session using a session handle
+     */
+    resumeSession(handle: string, context?: Array<{
+        role: string;
+        content: string;
+    }>): Promise<void>;
+    /**
+     * Update session configuration during an active session
+     * Allows dynamic updates to voice, instructions, tools, and other settings
+     *
+     * @param config Partial configuration to update
+     * @throws Error if not connected or update fails
+     *
+     * @example
+     * ```typescript
+     * // Change voice during conversation
+     * await voice.updateSessionConfig({
+     *   speaker: 'Charon'
+     * });
+     *
+     * // Update instructions
+     * await voice.updateSessionConfig({
+     *   instructions: 'You are now a helpful coding assistant'
+     * });
+     *
+     * // Add or update tools
+     * await voice.updateSessionConfig({
+     *   tools: [{ name: 'new_tool', ... }]
+     * });
+     * ```
+     */
+    updateSessionConfig(config: Partial<GeminiLiveVoiceConfig>): Promise<void>;
+    /**
+     * Get current connection state
+     */
+    getConnectionState(): 'disconnected' | 'connected';
+    /**
+     * Check if currently connected
+     */
+    isConnected(): boolean;
+    /**
+     * Get current speaker stream for audio concatenation
+     * This allows external access to the current audio stream being built
+     */
+    getCurrentSpeakerStream(): NodeJS.ReadableStream | null;
+    /**
+     * Get session handle for resumption
+     */
+    getSessionHandle(): string | undefined;
+    /**
+     * Get comprehensive session information
+     */
+    getSessionInfo(): {
+        id?: string;
+        handle?: string;
+        startTime?: Date;
+        duration?: number;
+        state: string;
+        config?: GeminiSessionConfig;
+        contextSize: number;
+    };
+    /**
+     * Get session context history
+     */
+    getContextHistory(): Array<{
+        role: string;
+        content: string;
+        timestamp: number;
+    }>;
+    /**
+     * Add to context history for session continuity
+     */
+    addToContext(role: 'user' | 'assistant', content: string): void;
+    /**
+     * Clear session context
+     */
+    clearContext(): void;
+    /**
+     * Enable or disable automatic reconnection
+     */
+    setAutoReconnect(enabled: boolean): void;
+    /**
+     * Send session resumption message
+     * @private
+     */
+    private sendSessionResumption;
+    /**
+     * Start monitoring session duration
+     * @private
+     */
+    private startSessionDurationMonitor;
+    /**
+     * Parse duration string to milliseconds
+     * @private
+     */
+    private parseDuration;
+    /**
+     * Compress context history to manage memory
+     * @private
+     */
+    private compressContext;
+    /**
+     * Setup WebSocket event listeners for Gemini Live API messages
+     * @private
+     */
+    private setupEventListeners;
+    /**
+     * Handle different types of messages from Gemini Live API
+     * @private
+     */
+    private handleGeminiMessage;
+    /**
+     * Handle setup completion message
+     * @private
+     */
+    private handleSetupComplete;
+    /**
+     * Handle session update confirmation
+     * @private
+     */
+    private handleSessionUpdated;
+    /**
+     * Handle server content (text/audio responses)
+     * @private
+     */
+    private handleServerContent;
+    /**
+     * Handle tool call requests from the model
+     * @private
+     */
+    private handleToolCall;
+    /**
+     * Handle token usage information
+     * @private
+     */
+    private handleUsageUpdate;
+    /**
+     * Handle session end
+     * @private
+     */
+    private handleSessionEnd;
+    /**
+     * Handle errors
+     * @private
+     */
+    private handleError;
+    /**
+     * Determine the modality from message data
+     * @private
+     */
+    private determineModality;
+    /**
+     * Send initial configuration to Gemini Live API
+     * @private
+     */
+    private sendInitialConfig;
+    /**
+     * Wait for Gemini Live session to be created and ready
+     * @private
+     */
+    private waitForSessionCreated;
+    /**
+     * Get OAuth access token for Vertex AI authentication
+     * Implements token caching and automatic refresh
+     * @private
+     */
+    private getAccessToken;
+    /**
+     * Get the current response ID from the server message
+     * This is needed to associate audio chunks with their respective responses.
+     * @private
+     */
+    private getCurrentResponseId;
+    /**
+     * Set the current response ID for the next audio chunk.
+     * This is used to track the response ID for the current turn.
+     * @private
+     */
+    private setCurrentResponseId;
+    /**
+     * Send an event to the Gemini Live API with queueing support
+     * @private
+     */
+    private sendEvent;
+    /**
+     * Equip the voice provider with tools
+     * @param tools Object containing tool definitions that can be called by the voice model
+     *
+     * @example
+     * ```typescript
+     * const weatherTool = createTool({
+     *   id: "getWeather",
+     *   description: "Get the current weather for a location",
+     *   inputSchema: z.object({
+     *     location: z.string().describe("The city and state, e.g. San Francisco, CA"),
+     *   }),
+     *   execute: async ({ context }) => {
+     *     // Fetch weather data from an API
+     *     const response = await fetch(
+     *       `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
+     *     );
+     *     const data = await response.json();
+     *     return {
+     *       message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
+     *     };
+     *   },
+     * });
+     *
+     * voice.addTools({
+     *   getWeather: weatherTool,
+     * });
+     * ```
+     */
+    addTools(tools: ToolsInput): void;
+    /**
+     * Get the current tools configured for this voice instance
+     * @returns Object containing the current tools
+     */
+    getTools(): ToolsInput | undefined;
+    private log;
+    /**
+     * Convert Zod schema to JSON Schema for tool parameters
+     * @private
+     */
+    private convertZodSchemaToJsonSchema;
+    /**
+     * Convert Zod definition to JSON Schema
+     * @private
+     */
+    private convertZodDefToJsonSchema;
+    /**
+     * Close the connection (alias for disconnect)
+     */
+    close(): void;
+    /**
+     * Trigger voice provider to respond
+     */
+    answer(_options?: Record<string, unknown>): Promise<void>;
+    /**
+     * Equip the voice provider with instructions
+     * @param instructions Instructions to add
+     */
+    addInstructions(instructions?: string): void;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAyF/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2EnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAmCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAuFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAY9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAiH3B;;;OAGG;YACW,cAAc;IAoF5B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IA+BjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,QAAQ,IAAI,UAAU,GAAG,SAAS;IAIlC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}