@mastra/voice-openai-realtime 0.0.0-vnextWorkflows-20250416071310 → 0.0.0-vnextWorkflows-20250417075051

package/CHANGELOG.md

@@ -1,17 +1,47 @@
 # @mastra/voice-openai-realtime
 
-## 0.0.0-vnextWorkflows-20250416071310
+## 0.0.0-vnextWorkflows-20250417075051
 
 ### Minor Changes
 
+- 9aaa64b: Don't connect the ws connection until connect is called
+- 1ef0f1f: Disconnect
 - 185f8e9: Emit add tools and silence warnings
 
 ### Patch Changes
 
+- 6cf1417: Voice reference and tool-call-start
+- Updated dependencies [9ee4293]
 - Updated dependencies [03f3cd0]
 - Updated dependencies [8a8a73b]
+- Updated dependencies [fe3ae4d]
 - Updated dependencies [6f92295]
-  - @mastra/core@0.0.0-vnextWorkflows-20250416071310
+  - @mastra/core@0.0.0-vnextWorkflows-20250417075051
+
+## 0.2.0-alpha.5
+
+### Patch Changes
+
+- 6cf1417: Voice reference and tool-call-start
+- Updated dependencies [fe3ae4d]
+  - @mastra/core@0.9.0-alpha.3
+
+## 0.2.0-alpha.4
+
+### Minor Changes
+
+- 1ef0f1f: Disconnect
+
+## 0.2.0-alpha.3
+
+### Minor Changes
+
+- 9aaa64b: Don't connect the ws connection until connect is called
+
+### Patch Changes
+
+- Updated dependencies [9ee4293]
+  - @mastra/core@0.8.4-alpha.2
 
 ## 0.2.0-alpha.2
 

package/README.md

@@ -20,6 +20,7 @@ OPENAI_API_KEY=your_api_key
 
 ```typescript
 import { OpenAIRealtimeVoice } from '@mastra/voice-openai-realtime';
+import { getMicrophoneStream } from '@mastra/node-audio';
 
 // Create a voice instance with default configuration
 const voice = new OpenAIRealtimeVoice();

package/dist/_tsup-dts-rollup.d.cts

@@ -43,7 +43,8 @@ export declare type OpenAIExecuteFunction = (args: any) => Promise<any>;
  * ```
  */
 export declare class OpenAIRealtimeVoice extends MastraVoice {
-    private ws;
+    private options;
+    private ws?;
     private state;
     private client;
     private events;
@@ -220,6 +221,7 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * ```
      */
     connect(): Promise<void>;
+    disconnect(): void;
     /**
      * Streams audio data in real-time to the OpenAI service.
      * Useful for continuous audio streaming scenarios like live microphone input.

package/dist/_tsup-dts-rollup.d.ts

@@ -43,7 +43,8 @@ export declare type OpenAIExecuteFunction = (args: any) => Promise<any>;
  * ```
  */
 export declare class OpenAIRealtimeVoice extends MastraVoice {
-    private ws;
+    private options;
+    private ws?;
     private state;
     private client;
     private events;
@@ -220,6 +221,7 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * ```
      */
     connect(): Promise<void>;
+    disconnect(): void;
     /**
      * Streams audio data in real-time to the OpenAI service.
      * Useful for continuous audio streaming scenarios like live microphone input.

package/dist/index.cjs

@@ -76,15 +76,6 @@ var DEFAULT_URL = "wss://api.openai.com/v1/realtime";
 var DEFAULT_MODEL = "gpt-4o-mini-realtime-preview-2024-12-17";
 var VOICES = ["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
 var OpenAIRealtimeVoice = class extends voice.MastraVoice {
-  ws;
-  state;
-  client;
-  events;
-  instructions;
-  tools;
-  debug;
-  queue = [];
-  transcriber;
   /**
    * Creates a new instance of OpenAIRealtimeVoice.
    *
@@ -108,22 +99,23 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    */
   constructor(options = {}) {
     super();
-    const url = `${options.url || DEFAULT_URL}?model=${options.model || DEFAULT_MODEL}`;
-    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;
-    this.ws = new ws.WebSocket(url, void 0, {
-      headers: {
-        Authorization: "Bearer " + apiKey,
-        "OpenAI-Beta": "realtime=v1"
-      }
-    });
+    this.options = options;
     this.client = new events.EventEmitter();
     this.state = "close";
     this.events = {};
     this.speaker = options.speaker || DEFAULT_VOICE;
     this.transcriber = options.transcriber || DEFAULT_TRANSCRIBER;
     this.debug = options.debug || false;
-    this.setupEventListeners();
   }
+  ws;
+  state;
+  client;
+  events;
+  instructions;
+  tools;
+  debug;
+  queue = [];
+  transcriber;
   /**
    * Returns a list of available voice speakers.
    *
@@ -300,7 +292,7 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
   }
   waitForOpen() {
     return new Promise((resolve) => {
-      this.ws.on("open", resolve);
+      this.ws?.on("open", resolve);
     });
   }
   waitForSessionCreated() {
@@ -321,8 +313,16 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * ```
    */
   async connect() {
-    await this.waitForOpen();
-    await this.waitForSessionCreated();
+    const url = `${this.options.url || DEFAULT_URL}?model=${this.options.model || DEFAULT_MODEL}`;
+    const apiKey = this.options.apiKey || process.env.OPENAI_API_KEY;
+    this.ws = new ws.WebSocket(url, void 0, {
+      headers: {
+        Authorization: "Bearer " + apiKey,
+        "OpenAI-Beta": "realtime=v1"
+      }
+    });
+    this.setupEventListeners();
+    await Promise.all([this.waitForOpen(), this.waitForSessionCreated()]);
     const openaiTools = transformTools(this.tools);
     this.updateConfig({
       instructions: this.instructions,
@@ -334,6 +334,10 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
     });
     this.state = "open";
   }
+  disconnect() {
+    this.state = "close";
+    this.ws?.close();
+  }
   /**
    * Streams audio data in real-time to the OpenAI service.
    * Useful for continuous audio streaming scenarios like live microphone input.
@@ -466,6 +470,9 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
   }
   setupEventListeners() {
     const speakerStreams = /* @__PURE__ */ new Map();
+    if (!this.ws) {
+      throw new Error("WebSocket not initialized");
+    }
     this.ws.on("message", (message) => {
       const data = JSON.parse(message.toString());
       this.client.emit(data.type, data);
@@ -478,7 +485,7 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
       this.emit("session.created", ev);
       const queue = this.queue.splice(0, this.queue.length);
       for (const ev2 of queue) {
-        this.ws.send(JSON.stringify(ev2));
+        this.ws?.send(JSON.stringify(ev2));
       }
     });
     this.client.on("session.updated", (ev) => {
@@ -541,16 +548,25 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
         console.warn(`Tool "${output.name}" not found`);
         return;
       }
+      if (tool?.execute) {
+        this.emit("tool-call-start", {
+          toolCallId: output.call_id,
+          toolName: output.name,
+          toolDescription: tool.description,
+          args: context
+        });
+      }
       const result = await tool?.execute?.(
         { context },
         {
-          toolCallId: "unknown",
+          toolCallId: output.call_id,
           messages: []
         }
       );
-      this.emit("tool-result", {
+      this.emit("tool-call-result", {
         toolCallId: output.call_id,
         toolName: output.name,
+        toolDescription: tool.description,
         args: context,
         result
       });
@@ -589,10 +605,10 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
     return btoa(binary);
   }
   sendEvent(type, data) {
-    if (this.ws.readyState !== this.ws.OPEN) {
+    if (!this.ws || this.ws.readyState !== this.ws.OPEN) {
       this.queue.push({ type, ...data });
     } else {
-      this.ws.send(
+      this.ws?.send(
         JSON.stringify({
           type,
           ...data

package/dist/index.js

@@ -74,15 +74,6 @@ var DEFAULT_URL = "wss://api.openai.com/v1/realtime";
 var DEFAULT_MODEL = "gpt-4o-mini-realtime-preview-2024-12-17";
 var VOICES = ["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
 var OpenAIRealtimeVoice = class extends MastraVoice {
-  ws;
-  state;
-  client;
-  events;
-  instructions;
-  tools;
-  debug;
-  queue = [];
-  transcriber;
   /**
    * Creates a new instance of OpenAIRealtimeVoice.
    *
@@ -106,22 +97,23 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
    */
   constructor(options = {}) {
     super();
-    const url = `${options.url || DEFAULT_URL}?model=${options.model || DEFAULT_MODEL}`;
-    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;
-    this.ws = new WebSocket(url, void 0, {
-      headers: {
-        Authorization: "Bearer " + apiKey,
-        "OpenAI-Beta": "realtime=v1"
-      }
-    });
+    this.options = options;
     this.client = new EventEmitter();
     this.state = "close";
     this.events = {};
     this.speaker = options.speaker || DEFAULT_VOICE;
     this.transcriber = options.transcriber || DEFAULT_TRANSCRIBER;
     this.debug = options.debug || false;
-    this.setupEventListeners();
   }
+  ws;
+  state;
+  client;
+  events;
+  instructions;
+  tools;
+  debug;
+  queue = [];
+  transcriber;
   /**
    * Returns a list of available voice speakers.
    *
@@ -298,7 +290,7 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
   }
   waitForOpen() {
     return new Promise((resolve) => {
-      this.ws.on("open", resolve);
+      this.ws?.on("open", resolve);
     });
   }
   waitForSessionCreated() {
@@ -319,8 +311,16 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
    * ```
    */
   async connect() {
-    await this.waitForOpen();
-    await this.waitForSessionCreated();
+    const url = `${this.options.url || DEFAULT_URL}?model=${this.options.model || DEFAULT_MODEL}`;
+    const apiKey = this.options.apiKey || process.env.OPENAI_API_KEY;
+    this.ws = new WebSocket(url, void 0, {
+      headers: {
+        Authorization: "Bearer " + apiKey,
+        "OpenAI-Beta": "realtime=v1"
+      }
+    });
+    this.setupEventListeners();
+    await Promise.all([this.waitForOpen(), this.waitForSessionCreated()]);
     const openaiTools = transformTools(this.tools);
     this.updateConfig({
       instructions: this.instructions,
@@ -332,6 +332,10 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
     });
     this.state = "open";
   }
+  disconnect() {
+    this.state = "close";
+    this.ws?.close();
+  }
   /**
    * Streams audio data in real-time to the OpenAI service.
    * Useful for continuous audio streaming scenarios like live microphone input.
@@ -464,6 +468,9 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
   }
   setupEventListeners() {
     const speakerStreams = /* @__PURE__ */ new Map();
+    if (!this.ws) {
+      throw new Error("WebSocket not initialized");
+    }
     this.ws.on("message", (message) => {
       const data = JSON.parse(message.toString());
       this.client.emit(data.type, data);
@@ -476,7 +483,7 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
       this.emit("session.created", ev);
       const queue = this.queue.splice(0, this.queue.length);
       for (const ev2 of queue) {
-        this.ws.send(JSON.stringify(ev2));
+        this.ws?.send(JSON.stringify(ev2));
       }
     });
     this.client.on("session.updated", (ev) => {
@@ -539,16 +546,25 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
         console.warn(`Tool "${output.name}" not found`);
         return;
       }
+      if (tool?.execute) {
+        this.emit("tool-call-start", {
+          toolCallId: output.call_id,
+          toolName: output.name,
+          toolDescription: tool.description,
+          args: context
+        });
+      }
       const result = await tool?.execute?.(
         { context },
         {
-          toolCallId: "unknown",
+          toolCallId: output.call_id,
           messages: []
         }
       );
-      this.emit("tool-result", {
+      this.emit("tool-call-result", {
         toolCallId: output.call_id,
         toolName: output.name,
+        toolDescription: tool.description,
         args: context,
         result
       });
@@ -587,10 +603,10 @@ var OpenAIRealtimeVoice = class extends MastraVoice {
     return btoa(binary);
   }
   sendEvent(type, data) {
-    if (this.ws.readyState !== this.ws.OPEN) {
+    if (!this.ws || this.ws.readyState !== this.ws.OPEN) {
       this.queue.push({ type, ...data });
     } else {
-      this.ws.send(
+      this.ws?.send(
         JSON.stringify({
           type,
           ...data

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/voice-openai-realtime",
-  "version": "0.0.0-vnextWorkflows-20250416071310",
+  "version": "0.0.0-vnextWorkflows-20250417075051",
   "description": "Mastra OpenAI Realtime API integration",
   "type": "module",
   "main": "dist/index.js",
@@ -23,7 +23,7 @@
     "openai-realtime-api": "^1.0.7",
     "ws": "^8.18.1",
     "zod-to-json-schema": "^3.24.5",
-    "@mastra/core": "0.0.0-vnextWorkflows-20250416071310"
+    "@mastra/core": "0.0.0-vnextWorkflows-20250417075051"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.2",

package/src/index.ts

@@ -106,7 +106,7 @@ type RealtimeClientServerEventMap = {
  * ```
  */
 export class OpenAIRealtimeVoice extends MastraVoice {
-  private ws: WebSocket;
+  private ws?: WebSocket;
   private state: 'close' | 'open';
   private client: EventEmitter<RealtimeClientServerEventMap>;
   private events: EventMap;
@@ -138,7 +138,7 @@ export class OpenAIRealtimeVoice extends MastraVoice {
    * ```
    */
   constructor(
-    options: {
+    private options: {
       model?: string;
       url?: string;
       apiKey?: string;
@@ -149,22 +149,12 @@ export class OpenAIRealtimeVoice extends MastraVoice {
   ) {
     super();
 
-    const url = `${options.url || DEFAULT_URL}?model=${options.model || DEFAULT_MODEL}`;
-    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;
-    this.ws = new WebSocket(url, undefined, {
-      headers: {
-        Authorization: 'Bearer ' + apiKey,
-        'OpenAI-Beta': 'realtime=v1',
-      },
-    });
-
     this.client = new EventEmitter();
     this.state = 'close';
     this.events = {} as EventMap;
     this.speaker = options.speaker || DEFAULT_VOICE;
     this.transcriber = options.transcriber || DEFAULT_TRANSCRIBER;
     this.debug = options.debug || false;
-    this.setupEventListeners();
   }
 
   /**
@@ -355,7 +345,7 @@ export class OpenAIRealtimeVoice extends MastraVoice {
 
   waitForOpen() {
     return new Promise(resolve => {
-      this.ws.on('open', resolve);
+      this.ws?.on('open', resolve);
     });
   }
 
@@ -378,8 +368,17 @@ export class OpenAIRealtimeVoice extends MastraVoice {
    * ```
    */
   async connect() {
-    await this.waitForOpen();
-    await this.waitForSessionCreated();
+    const url = `${this.options.url || DEFAULT_URL}?model=${this.options.model || DEFAULT_MODEL}`;
+    const apiKey = this.options.apiKey || process.env.OPENAI_API_KEY;
+    this.ws = new WebSocket(url, undefined, {
+      headers: {
+        Authorization: 'Bearer ' + apiKey,
+        'OpenAI-Beta': 'realtime=v1',
+      },
+    });
+
+    this.setupEventListeners();
+    await Promise.all([this.waitForOpen(), this.waitForSessionCreated()]);
 
     const openaiTools = transformTools(this.tools);
     this.updateConfig({
@@ -393,6 +392,11 @@ export class OpenAIRealtimeVoice extends MastraVoice {
     this.state = 'open';
   }
 
+  disconnect() {
+    this.state = 'close';
+    this.ws?.close();
+  }
+
   /**
    * Streams audio data in real-time to the OpenAI service.
    * Useful for continuous audio streaming scenarios like live microphone input.
@@ -534,6 +538,10 @@ export class OpenAIRealtimeVoice extends MastraVoice {
   private setupEventListeners(): void {
     const speakerStreams = new Map<string, StreamWithId>();
 
+    if (!this.ws) {
+      throw new Error('WebSocket not initialized');
+    }
+
     this.ws.on('message', message => {
       const data = JSON.parse(message.toString());
       this.client.emit(data.type, data);
@@ -549,7 +557,7 @@ export class OpenAIRealtimeVoice extends MastraVoice {
 
       const queue = this.queue.splice(0, this.queue.length);
       for (const ev of queue) {
-        this.ws.send(JSON.stringify(ev));
+        this.ws?.send(JSON.stringify(ev));
       }
     });
     this.client.on('session.updated', ev => {
@@ -619,19 +627,32 @@ export class OpenAIRealtimeVoice extends MastraVoice {
         console.warn(`Tool "${output.name}" not found`);
         return;
       }
+
+      if (tool?.execute) {
+        this.emit('tool-call-start', {
+          toolCallId: output.call_id,
+          toolName: output.name,
+          toolDescription: tool.description,
+          args: context,
+        });
+      }
+
       const result = await tool?.execute?.(
         { context },
         {
-          toolCallId: 'unknown',
+          toolCallId: output.call_id,
           messages: [],
         },
       );
-      this.emit('tool-result', {
+
+      this.emit('tool-call-result', {
         toolCallId: output.call_id,
         toolName: output.name,
+        toolDescription: tool.description,
         args: context,
         result,
       });
+
       this.sendEvent('conversation.item.create', {
         item: {
           type: 'function_call_output',
@@ -669,10 +690,10 @@ export class OpenAIRealtimeVoice extends MastraVoice {
   }
 
   private sendEvent(type: string, data: any) {
-    if (this.ws.readyState !== this.ws.OPEN) {
+    if (!this.ws || this.ws.readyState !== this.ws.OPEN) {
       this.queue.push({ type: type, ...data });
     } else {
-      this.ws.send(
+      this.ws?.send(
         JSON.stringify({
           type: type,
           ...data,