@mastra/voice-google-gemini-live 0.0.0-remove-unused-model-providers-api-20251030210744 → 0.0.0-remove-ai-peer-dep-from-evals-20260105220639

package/dist/docs/voice/02-speech-to-speech.md

@@ -0,0 +1,106 @@
+> Overview of speech-to-speech capabilities in Mastra, including real-time interactions and event-driven architecture.
+
+# Speech-to-Speech Capabilities in Mastra
+
+## Introduction
+
+Speech-to-Speech (STS) in Mastra provides a standardized interface for real-time interactions across multiple providers.
+STS enables continuous bidirectional audio communication through listening to events from Realtime models. Unlike separate TTS and STT operations, STS maintains an open connection that processes speech continuously in both directions.
+
+## Configuration
+
+- **`apiKey`**: Your OpenAI API key. Falls back to the `OPENAI_API_KEY` environment variable.
+- **`model`**: The model ID to use for real-time voice interactions (e.g., `gpt-5.1-realtime`).
+- **`speaker`**: The default voice ID for speech synthesis. This allows you to specify which voice to use for the speech output.
+
+```typescript
+const voice = new OpenAIRealtimeVoice({
+  apiKey: "your-openai-api-key",
+  model: "gpt-5.1-realtime",
+  speaker: "alloy", // Default voice
+});
+
+// If using default settings the configuration can be simplified to:
+const voice = new OpenAIRealtimeVoice();
+```
+
+## Using STS
+
+```typescript
+import { Agent } from "@mastra/core/agent";
+import { OpenAIRealtimeVoice } from "@mastra/voice-openai-realtime";
+import { playAudio, getMicrophoneStream } from "@mastra/node-audio";
+
+const agent = new Agent({
+  id: "agent",
+  name: "OpenAI Realtime Agent",
+  instructions: `You are a helpful assistant with real-time voice capabilities.`,
+  model: "openai/gpt-5.1",
+  voice: new OpenAIRealtimeVoice(),
+});
+
+// Connect to the voice service
+await agent.voice.connect();
+
+// Listen for agent audio responses
+agent.voice.on("speaker", ({ audio }) => {
+  playAudio(audio);
+});
+
+// Initiate the conversation
+await agent.voice.speak("How can I help you today?");
+
+// Send continuous audio from the microphone
+const micStream = getMicrophoneStream();
+await agent.voice.send(micStream);
+```
+
+For integrating Speech-to-Speech capabilities with agents, refer to the [Adding Voice to Agents](https://mastra.ai/docs/v1/agents/adding-voice) documentation.
+
+## Google Gemini Live (Realtime)
+
+```typescript
+import { Agent } from "@mastra/core/agent";
+import { GeminiLiveVoice } from "@mastra/voice-google-gemini-live";
+import { playAudio, getMicrophoneStream } from "@mastra/node-audio";
+
+const agent = new Agent({
+  id: "agent",
+  name: "Gemini Live Agent",
+  instructions:
+    "You are a helpful assistant with real-time voice capabilities.",
+  // Model used for text generation; voice provider handles realtime audio
+  model: "openai/gpt-5.1",
+  voice: new GeminiLiveVoice({
+    apiKey: process.env.GOOGLE_API_KEY,
+    model: "gemini-2.0-flash-exp",
+    speaker: "Puck",
+    debug: true,
+    // Vertex AI option:
+    // vertexAI: true,
+    // project: 'your-gcp-project',
+    // location: 'us-central1',
+    // serviceAccountKeyFile: '/path/to/service-account.json',
+  }),
+});
+
+await agent.voice.connect();
+
+agent.voice.on("speaker", ({ audio }) => {
+  playAudio(audio);
+});
+
+agent.voice.on("writing", ({ role, text }) => {
+  console.log(`${role}: ${text}`);
+});
+
+await agent.voice.speak("How can I help you today?");
+
+const micStream = getMicrophoneStream();
+await agent.voice.send(micStream);
+```
+
+Note:
+
+- Live API requires `GOOGLE_API_KEY`. Vertex AI requires project/location and service account credentials.
+- Events: `speaker` (audio stream), `writing` (text), `turnComplete`, `usage`, and `error`.

package/dist/docs/voice/03-reference.md

@@ -0,0 +1,303 @@
+# Voice API Reference
+
+> API reference for voice - 1 entries
+
+
+---
+
+## Reference: Google Gemini Live Voice
+
+> Documentation for the GeminiLiveVoice class, providing real-time multimodal voice interactions using Google
+
+The GeminiLiveVoice class provides real-time voice interaction capabilities using Google's Gemini Live API. It supports bidirectional audio streaming, tool calling, session management, and both standard Google API and Vertex AI authentication methods.
+
+## Usage Example
+
+```typescript
+import { GeminiLiveVoice } from "@mastra/voice-google-gemini-live";
+import { playAudio, getMicrophoneStream } from "@mastra/node-audio";
+
+// Initialize with Gemini API (using API key)
+const voice = new GeminiLiveVoice({
+  apiKey: process.env.GOOGLE_API_KEY, // Required for Gemini API
+  model: "gemini-2.0-flash-exp",
+  speaker: "Puck", // Default voice
+  debug: true,
+});
+
+// Or initialize with Vertex AI (using OAuth)
+const voiceWithVertexAI = new GeminiLiveVoice({
+  vertexAI: true,
+  project: "your-gcp-project",
+  location: "us-central1",
+  serviceAccountKeyFile: "/path/to/service-account.json",
+  model: "gemini-2.0-flash-exp",
+  speaker: "Puck",
+});
+
+// Or use the VoiceConfig pattern (recommended for consistency with other providers)
+const voiceWithConfig = new GeminiLiveVoice({
+  speechModel: {
+    name: "gemini-2.0-flash-exp",
+    apiKey: process.env.GOOGLE_API_KEY,
+  },
+  speaker: "Puck",
+  realtimeConfig: {
+    model: "gemini-2.0-flash-exp",
+    apiKey: process.env.GOOGLE_API_KEY,
+    options: {
+      debug: true,
+      sessionConfig: {
+        interrupts: { enabled: true },
+      },
+    },
+  },
+});
+
+// Establish connection (required before using other methods)
+await voice.connect();
+
+// Set up event listeners
+voice.on("speaker", (audioStream) => {
+  // Handle audio stream (NodeJS.ReadableStream)
+  playAudio(audioStream);
+});
+
+voice.on("writing", ({ text, role }) => {
+  // Handle transcribed text
+  console.log(`${role}: ${text}`);
+});
+
+voice.on("turnComplete", ({ timestamp }) => {
+  // Handle turn completion
+  console.log("Turn completed at:", timestamp);
+});
+
+// Convert text to speech
+await voice.speak("Hello, how can I help you today?", {
+  speaker: "Charon", // Override default voice
+  responseModalities: ["AUDIO", "TEXT"],
+});
+
+// Process audio input
+const microphoneStream = getMicrophoneStream();
+await voice.send(microphoneStream);
+
+// Update session configuration
+await voice.updateSessionConfig({
+  speaker: "Kore",
+  instructions: "Be more concise in your responses",
+});
+
+// When done, disconnect
+await voice.disconnect();
+// Or use the synchronous wrapper
+voice.close();
+```
+
+## Configuration
+
+### Constructor Options
+
+### Session Configuration
+
+## Methods
+
+### connect()
+
+Establishes a connection to the Gemini Live API. Must be called before using speak, listen, or send methods.
+
+### speak()
+
+Converts text to speech and sends it to the model. Can accept either a string or a readable stream as input.
+
+Returns: `Promise<void>` (responses are emitted via `speaker` and `writing` events)
+
+### listen()
+
+Processes audio input for speech recognition. Takes a readable stream of audio data and returns the transcribed text.
+
+Returns: `Promise<string>` - The transcribed text
+
+### send()
+
+Streams audio data in real-time to the Gemini service for continuous audio streaming scenarios like live microphone input.
+
+Returns: `Promise<void>`
+
+### updateSessionConfig()
+
+Updates the session configuration dynamically. This can be used to modify voice settings, speaker selection, and other runtime configurations.
+
+Returns: `Promise<void>`
+
+### addTools()
+
+Adds a set of tools to the voice instance. Tools allow the model to perform additional actions during conversations. When GeminiLiveVoice is added to an Agent, any tools configured for the Agent will automatically be available to the voice interface.
+
+Returns: `void`
+
+### addInstructions()
+
+Adds or updates system instructions for the model.
+
+Returns: `void`
+
+### answer()
+
+Triggers a response from the model. This method is primarily used internally when integrated with an Agent.
+
+Returns: `Promise<void>`
+
+### getSpeakers()
+
+Returns a list of available voice speakers for the Gemini Live API.
+
+Returns: `Promise<Array<{ voiceId: string; description?: string }>>`
+
+### disconnect()
+
+Disconnects from the Gemini Live session and cleans up resources. This is the async method that properly handles cleanup.
+
+Returns: `Promise<void>`
+
+### close()
+
+Synchronous wrapper for disconnect(). Calls disconnect() internally without awaiting.
+
+Returns: `void`
+
+### on()
+
+Registers an event listener for voice events.
+
+Returns: `void`
+
+### off()
+
+Removes a previously registered event listener.
+
+Returns: `void`
+
+## Events
+
+The GeminiLiveVoice class emits the following events:
+
+## Available Models
+
+The following Gemini Live models are available:
+
+- `gemini-2.0-flash-exp` (default)
+- `gemini-2.0-flash-exp-image-generation`
+- `gemini-2.0-flash-live-001`
+- `gemini-live-2.5-flash-preview-native-audio`
+- `gemini-2.5-flash-exp-native-audio-thinking-dialog`
+- `gemini-live-2.5-flash-preview`
+- `gemini-2.6.flash-preview-tts`
+
+## Available Voices
+
+The following voice options are available:
+
+- `Puck` (default): Conversational, friendly
+- `Charon`: Deep, authoritative
+- `Kore`: Neutral, professional
+- `Fenrir`: Warm, approachable
+
+## Authentication Methods
+
+### Gemini API (Development)
+
+The simplest method using an API key from [Google AI Studio](https://makersuite.google.com/app/apikey):
+
+```typescript
+const voice = new GeminiLiveVoice({
+  apiKey: "your-api-key", // Required for Gemini API
+  model: "gemini-2.0-flash-exp",
+});
+```
+
+### Vertex AI (Production)
+
+For production use with OAuth authentication and Google Cloud Platform:
+
+```typescript
+// Using service account key file
+const voice = new GeminiLiveVoice({
+  vertexAI: true,
+  project: "your-gcp-project",
+  location: "us-central1",
+  serviceAccountKeyFile: "/path/to/service-account.json",
+});
+
+// Using Application Default Credentials
+const voice = new GeminiLiveVoice({
+  vertexAI: true,
+  project: "your-gcp-project",
+  location: "us-central1",
+});
+
+// Using service account impersonation
+const voice = new GeminiLiveVoice({
+  vertexAI: true,
+  project: "your-gcp-project",
+  location: "us-central1",
+  serviceAccountEmail: "service-account@project.iam.gserviceaccount.com",
+});
+```
+
+## Advanced Features
+
+### Session Management
+
+The Gemini Live API supports session resumption for handling network interruptions:
+
+```typescript
+voice.on("sessionHandle", ({ handle, expiresAt }) => {
+  // Store session handle for resumption
+  saveSessionHandle(handle, expiresAt);
+});
+
+// Resume a previous session
+const voice = new GeminiLiveVoice({
+  sessionConfig: {
+    enableResumption: true,
+    maxDuration: "2h",
+  },
+});
+```
+
+### Tool Calling
+
+Enable the model to call functions during conversations:
+
+```typescript
+import { z } from "zod";
+
+voice.addTools({
+  weather: {
+    description: "Get weather information",
+    parameters: z.object({
+      location: z.string(),
+    }),
+    execute: async ({ location }) => {
+      const weather = await getWeather(location);
+      return weather;
+    },
+  },
+});
+
+voice.on("toolCall", ({ name, args, id }) => {
+  console.log(`Tool called: ${name} with args:`, args);
+});
+```
+
+## Notes
+
+- The Gemini Live API uses WebSockets for real-time communication
+- Audio is processed as 16kHz PCM16 for input and 24kHz PCM16 for output
+- The voice instance must be connected with `connect()` before using other methods
+- Always call `close()` when done to properly clean up resources
+- Vertex AI authentication requires appropriate IAM permissions (`aiplatform.user` role)
+- Session resumption allows recovery from network interruptions
+- The API supports real-time interactions with text and audio

package/dist/index.cjs

@@ -1510,7 +1510,8 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
       let wsUrl;
       let headers = {};
       if (this.options.vertexAI) {
-        wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
+        const location = this.getVertexLocation();
+        wsUrl = `wss://${location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent`;
         await this.authManager.initialize();
         const accessToken = await this.authManager.getAccessToken();
         headers = { headers: { Authorization: `Bearer ${accessToken}` } };
@@ -2259,6 +2260,18 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
             role: "assistant"
           });
         }
+        if (part.functionCall) {
+          this.log("Found function call in serverContent.modelTurn.parts", part.functionCall);
+          const toolCallData = {
+            toolCall: {
+              name: part.functionCall.name,
+              args: part.functionCall.args || {},
+              id: part.functionCall.id || crypto.randomUUID()
+            }
+          };
+          void this.handleToolCall(toolCallData);
+          continue;
+        }
         if (part.inlineData?.mimeType?.includes("audio") && typeof part.inlineData.data === "string") {
           try {
             const audioData = part.inlineData.data;
@@ -2333,9 +2346,24 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
     if (!data.toolCall) {
       return;
     }
-    const toolName = data.toolCall.name || "";
-    const toolArgs = data.toolCall.args || {};
-    const toolId = data.toolCall.id || crypto.randomUUID();
+    let toolCalls = [];
+    if (data.toolCall.functionCalls && Array.isArray(data.toolCall.functionCalls)) {
+      toolCalls = data.toolCall.functionCalls;
+    } else if (data.toolCall.name) {
+      toolCalls = [{ name: data.toolCall.name, args: data.toolCall.args, id: data.toolCall.id }];
+    }
+    for (const toolCall of toolCalls) {
+      const toolName = toolCall.name || "";
+      const toolArgs = toolCall.args || {};
+      const toolId = toolCall.id || crypto.randomUUID();
+      await this.processSingleToolCall(toolName, toolArgs, toolId);
+    }
+  }
+  /**
+   * Process a single tool call
+   * @private
+   */
+  async processSingleToolCall(toolName, toolArgs, toolId) {
     this.log("Processing tool call", { toolName, toolArgs, toolId });
     this.emit("toolCall", {
       name: toolName,
@@ -2355,36 +2383,38 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
       let result;
       if (tool.execute) {
         this.log("Executing tool", { toolName, toolArgs });
-        result = await tool.execute(
-          { context: toolArgs, requestContext: this.requestContext },
-          {
-            toolCallId: toolId,
-            messages: []
-          }
-        );
+        result = await tool.execute(toolArgs, { requestContext: this.requestContext });
         this.log("Tool executed successfully", { toolName, result });
       } else {
         this.log("Tool has no execute function", { toolName });
         result = { error: "Tool has no execute function" };
       }
       const toolResultMessage = {
-        tool_result: {
-          tool_call_id: toolId,
-          result
+        toolResponse: {
+          functionResponses: [
+            {
+              id: toolId,
+              response: result
+            }
+          ]
         }
       };
-      this.sendEvent("tool_result", toolResultMessage);
+      this.sendEvent("toolResponse", toolResultMessage);
       this.log("Tool result sent", { toolName, toolId, result });
     } catch (error) {
       const errorMessage = error instanceof Error ? error.message : "Unknown error";
       this.log("Tool execution failed", { toolName, error: errorMessage });
       const errorResultMessage = {
-        tool_result: {
-          tool_call_id: toolId,
-          result: { error: errorMessage }
+        toolResponse: {
+          functionResponses: [
+            {
+              id: toolId,
+              response: { error: errorMessage }
+            }
+          ]
         }
       };
-      this.sendEvent("tool_result", errorResultMessage);
+      this.sendEvent("toolResponse", errorResultMessage);
       this.createAndEmitError("tool_execution_error" /* TOOL_EXECUTION_ERROR */, `Tool execution failed: ${errorMessage}`, {
         toolName,
         toolArgs,
@@ -2444,6 +2474,31 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
     }
     return "text";
   }
+  /**
+   * Resolve Vertex AI location with sensible default
+   * @private
+   */
+  getVertexLocation() {
+    return this.options.location?.trim() || "us-central1";
+  }
+  /**
+   * Resolve the correct model identifier for Gemini API or Vertex AI
+   * @private
+   */
+  resolveModelIdentifier() {
+    const model = this.options.model ?? DEFAULT_MODEL;
+    if (!this.options.vertexAI) {
+      return `models/${model}`;
+    }
+    if (!this.options.project) {
+      throw this.createAndEmitError(
+        "project_id_missing" /* PROJECT_ID_MISSING */,
+        "Google Cloud project ID is required when using Vertex AI."
+      );
+    }
+    const location = this.getVertexLocation();
+    return `projects/${this.options.project}/locations/${location}/publishers/google/models/${model}`;
+  }
   /**
    * Send initial configuration to Gemini Live API
    * @private
@@ -2454,7 +2509,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
     }
     const setupMessage = {
       setup: {
-        model: `models/${this.options.model}`
+        model: this.resolveModelIdentifier()
       }
     };
     if (this.options.instructions) {
@@ -2603,6 +2658,8 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
       message = data;
     } else if (type === "realtime_input" && data.realtime_input) {
       message = data;
+    } else if (type === "toolResponse" && data.toolResponse) {
+      message = data;
     } else if (type === "session.update" && data.session) {
       message = data;
     } else {
@@ -2628,14 +2685,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
    *   inputSchema: z.object({
    *     location: z.string().describe("The city and state, e.g. San Francisco, CA"),
    *   }),
-   *   execute: async ({ context }) => {
+   *   execute: async (inputData) => {
    *     // Fetch weather data from an API
    *     const response = await fetch(
-   *       `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
+   *       `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
    *     );
    *     const data = await response.json();
    *     return {
-   *       message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
+   *       message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
    *     };
    *   },
    * });