@mastra/voice-openai-realtime 0.0.5-alpha.0 → 0.1.0-alpha.2

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/voice-openai-realtime@0.0.5-alpha.0 build /home/runner/work/mastra/mastra/voice/openai-realtime-api
+> @mastra/voice-openai-realtime@0.1.0-alpha.2 build /home/runner/work/mastra/mastra/voice/openai-realtime-api
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 9719ms
+TSC ⚡️ Build success in 9123ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/voice/openai-realtime-api/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/voice/openai-realtime-api/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 10943ms
+DTS ⚡️ Build success in 10151ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-CJS dist/index.cjs 14.99 KB
-CJS ⚡️ Build success in 515ms
-ESM dist/index.js 14.90 KB
-ESM ⚡️ Build success in 515ms
+CJS dist/index.cjs 17.80 KB
+CJS ⚡️ Build success in 842ms
+ESM dist/index.js 17.75 KB
+ESM ⚡️ Build success in 843ms

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @mastra/voice-openai-realtime
 
+## 0.1.0-alpha.2
+
+### Patch Changes
+
+- Updated dependencies [a838fde]
+- Updated dependencies [a8bd4cf]
+- Updated dependencies [7a3eeb0]
+- Updated dependencies [6530ad1]
+  - @mastra/core@0.7.0-alpha.2
+
+## 0.1.0-alpha.1
+
+### Minor Changes
+
+- 443b118: This update removed an external dependency on an unmaintained package and implemented a native websocket connection.
+
+### Patch Changes
+
+- Updated dependencies [0b54522]
+- Updated dependencies [1af25d5]
+- Updated dependencies [27439ad]
+  - @mastra/core@0.7.0-alpha.1
+
 ## 0.0.5-alpha.0
 
 ### Patch Changes

package/dist/_tsup-dts-rollup.d.cts

@@ -43,10 +43,13 @@ export declare type OpenAIExecuteFunction = (args: any) => Promise<any>;
  * ```
  */
 export declare class OpenAIRealtimeVoice extends MastraVoice {
-    private client;
+    private ws;
     private state;
+    private client;
     private events;
-    tools?: TTools;
+    private instructions?;
+    private tools?;
+    private debug;
     /**
      * Creates a new instance of OpenAIRealtimeVoice.
      *
@@ -55,13 +58,8 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * @param options.chatModel.model - The model ID to use (defaults to GPT-4 Mini Realtime)
      * @param options.chatModel.apiKey - OpenAI API key. Falls back to process.env.OPENAI_API_KEY
      * @param options.chatModel.tools - Tools configuration for the model
-     * @param options.chatModel.options - Additional options for the realtime client
-     * @param options.chatModel.options.sessionConfig - Session configuration overrides
-     * @param options.chatModel.options.url - Custom WebSocket URL
-     * @param options.chatModel.options.dangerouslyAllowAPIKeyInBrowser - Whether to allow API key in browser
-     * @param options.chatModel.options.debug - Enable debug logging
-     * @param options.chatModel.options.tools - Additional tools configuration
      * @param options.speaker - Voice ID to use (defaults to 'alloy')
+     * @param options.debug - Enable debug mode
      *
      * @example
      * ```typescript
@@ -74,20 +72,16 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * });
      * ```
      */
-    constructor({ chatModel, speaker, }?: {
+    constructor({ chatModel, speaker, debug, }?: {
         chatModel?: {
             model?: string;
             apiKey?: string;
             tools?: TTools;
-            options?: {
-                sessionConfig?: Realtime.SessionConfig;
-                url?: string;
-                dangerouslyAllowAPIKeyInBrowser?: boolean;
-                debug?: boolean;
-                tools?: TTools;
-            };
+            instructions?: string;
+            url?: string;
         };
         speaker?: Realtime.Voice;
+        debug?: boolean;
     });
     /**
      * Returns a list of available voice speakers.
@@ -175,7 +169,7 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * });
      * ```
      */
-    updateConfig(sessionConfig: Realtime.SessionConfig): void;
+    updateConfig(sessionConfig: unknown): void;
     /**
      * Processes audio input for speech recognition.
      * Takes a readable stream of audio data and emits a writing event.
@@ -200,6 +194,8 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * ```
      */
     listen(audioData: NodeJS.ReadableStream): Promise<void>;
+    waitForOpen(): Promise<unknown>;
+    waitForSessionCreated(): Promise<unknown>;
     /**
      * Establishes a connection to the OpenAI realtime service.
      * Must be called before using speak, listen, or relay functions.
@@ -231,7 +227,7 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * await voice.relay(micStream);
      * ```
      */
-    send(audioData: NodeJS.ReadableStream | Int16Array): Promise<void>;
+    send(audioData: NodeJS.ReadableStream | Int16Array, eventId?: string): Promise<void>;
     /**
      * Sends a response to the OpenAI Realtime API.
      *
@@ -304,11 +300,15 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      */
     private emit;
     private setupEventListeners;
+    private handleFunctionCalls;
+    private handleFunctionCall;
     private int16ArrayToBase64;
+    private sendEvent;
 }
 
 export declare const transformTools: (tools?: TTools_2) => {
     openaiTool: {
+        type: string;
         name: string;
         description: string;
         parameters: {

package/dist/_tsup-dts-rollup.d.ts

@@ -43,10 +43,13 @@ export declare type OpenAIExecuteFunction = (args: any) => Promise<any>;
  * ```
  */
 export declare class OpenAIRealtimeVoice extends MastraVoice {
-    private client;
+    private ws;
     private state;
+    private client;
     private events;
-    tools?: TTools;
+    private instructions?;
+    private tools?;
+    private debug;
     /**
      * Creates a new instance of OpenAIRealtimeVoice.
      *
@@ -55,13 +58,8 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * @param options.chatModel.model - The model ID to use (defaults to GPT-4 Mini Realtime)
      * @param options.chatModel.apiKey - OpenAI API key. Falls back to process.env.OPENAI_API_KEY
      * @param options.chatModel.tools - Tools configuration for the model
-     * @param options.chatModel.options - Additional options for the realtime client
-     * @param options.chatModel.options.sessionConfig - Session configuration overrides
-     * @param options.chatModel.options.url - Custom WebSocket URL
-     * @param options.chatModel.options.dangerouslyAllowAPIKeyInBrowser - Whether to allow API key in browser
-     * @param options.chatModel.options.debug - Enable debug logging
-     * @param options.chatModel.options.tools - Additional tools configuration
      * @param options.speaker - Voice ID to use (defaults to 'alloy')
+     * @param options.debug - Enable debug mode
      *
      * @example
      * ```typescript
@@ -74,20 +72,16 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * });
      * ```
      */
-    constructor({ chatModel, speaker, }?: {
+    constructor({ chatModel, speaker, debug, }?: {
         chatModel?: {
             model?: string;
             apiKey?: string;
             tools?: TTools;
-            options?: {
-                sessionConfig?: Realtime.SessionConfig;
-                url?: string;
-                dangerouslyAllowAPIKeyInBrowser?: boolean;
-                debug?: boolean;
-                tools?: TTools;
-            };
+            instructions?: string;
+            url?: string;
         };
         speaker?: Realtime.Voice;
+        debug?: boolean;
     });
     /**
      * Returns a list of available voice speakers.
@@ -175,7 +169,7 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * });
      * ```
      */
-    updateConfig(sessionConfig: Realtime.SessionConfig): void;
+    updateConfig(sessionConfig: unknown): void;
     /**
      * Processes audio input for speech recognition.
      * Takes a readable stream of audio data and emits a writing event.
@@ -200,6 +194,8 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * ```
      */
     listen(audioData: NodeJS.ReadableStream): Promise<void>;
+    waitForOpen(): Promise<unknown>;
+    waitForSessionCreated(): Promise<unknown>;
     /**
      * Establishes a connection to the OpenAI realtime service.
      * Must be called before using speak, listen, or relay functions.
@@ -231,7 +227,7 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      * await voice.relay(micStream);
      * ```
      */
-    send(audioData: NodeJS.ReadableStream | Int16Array): Promise<void>;
+    send(audioData: NodeJS.ReadableStream | Int16Array, eventId?: string): Promise<void>;
     /**
      * Sends a response to the OpenAI Realtime API.
      *
@@ -304,11 +300,15 @@ export declare class OpenAIRealtimeVoice extends MastraVoice {
      */
     private emit;
     private setupEventListeners;
+    private handleFunctionCalls;
+    private handleFunctionCall;
     private int16ArrayToBase64;
+    private sendEvent;
 }
 
 export declare const transformTools: (tools?: TTools_2) => {
     openaiTool: {
+        type: string;
         name: string;
         description: string;
         parameters: {

package/dist/index.cjs

@@ -1,8 +1,9 @@
 'use strict';
 
-var voice = require('@mastra/core/voice');
-var openaiRealtimeApi = require('openai-realtime-api');
+var events = require('events');
 var stream = require('stream');
+var voice = require('@mastra/core/voice');
+var ws = require('ws');
 var zodToJsonSchema = require('zod-to-json-schema');
 
 // src/index.ts
@@ -29,6 +30,7 @@ var transformTools = (tools) => {
       continue;
     }
     const openaiTool = {
+      type: "function",
       name,
       description: tool.description || `Tool: ${name}`,
       parameters
@@ -66,19 +68,17 @@ var isReadableStream = (obj) => {
 
 // src/index.ts
 var DEFAULT_VOICE = "alloy";
+var DEFAULT_URL = "wss://api.openai.com/v1/realtime";
 var DEFAULT_MODEL = "gpt-4o-mini-realtime-preview-2024-12-17";
-var DEFAULT_VAD_CONFIG = {
-  type: "server_vad",
-  threshold: 0.5,
-  prefix_padding_ms: 1e3,
-  silence_duration_ms: 1e3
-};
 var VOICES = ["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
 var OpenAIRealtimeVoice = class extends voice.MastraVoice {
-  client;
+  ws;
   state;
+  client;
   events;
+  instructions;
   tools;
+  debug;
   /**
    * Creates a new instance of OpenAIRealtimeVoice.
    *
@@ -87,13 +87,8 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * @param options.chatModel.model - The model ID to use (defaults to GPT-4 Mini Realtime)
    * @param options.chatModel.apiKey - OpenAI API key. Falls back to process.env.OPENAI_API_KEY
    * @param options.chatModel.tools - Tools configuration for the model
-   * @param options.chatModel.options - Additional options for the realtime client
-   * @param options.chatModel.options.sessionConfig - Session configuration overrides
-   * @param options.chatModel.options.url - Custom WebSocket URL
-   * @param options.chatModel.options.dangerouslyAllowAPIKeyInBrowser - Whether to allow API key in browser
-   * @param options.chatModel.options.debug - Enable debug logging
-   * @param options.chatModel.options.tools - Additional tools configuration
    * @param options.speaker - Voice ID to use (defaults to 'alloy')
+   * @param options.debug - Enable debug mode
    *
    * @example
    * ```typescript
@@ -108,25 +103,26 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    */
   constructor({
     chatModel,
-    speaker
+    speaker,
+    debug = false
   } = {}) {
     super();
-    this.client = new openaiRealtimeApi.RealtimeClient({
-      apiKey: chatModel?.apiKey || process.env.OPENAI_API_KEY,
-      model: chatModel?.model || DEFAULT_MODEL,
-      ...chatModel?.options,
-      sessionConfig: {
-        voice: speaker || DEFAULT_VOICE,
-        turn_detection: DEFAULT_VAD_CONFIG,
-        ...chatModel?.options?.sessionConfig
+    const url = `${chatModel?.url || DEFAULT_URL}?model=${chatModel?.model || DEFAULT_MODEL}`;
+    const apiKey = chatModel?.apiKey || process.env.OPENAI_API_KEY;
+    this.ws = new ws.WebSocket(url, void 0, {
+      headers: {
+        Authorization: "Bearer " + apiKey,
+        "OpenAI-Beta": "realtime=v1"
       }
     });
+    this.client = new events.EventEmitter();
     this.state = "close";
     this.events = {};
+    this.tools = chatModel?.tools;
+    this.instructions = chatModel?.instructions;
+    this.speaker = speaker || DEFAULT_VOICE;
+    this.debug = debug;
     this.setupEventListeners();
-    if (chatModel?.tools) {
-      this.addTools(chatModel.tools);
-    }
   }
   /**
    * Returns a list of available voice speakers.
@@ -152,8 +148,8 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * ```
    */
   close() {
-    if (!this.client) return;
-    this.client.disconnect();
+    if (!this.ws) return;
+    this.ws.close();
     this.state = "close";
   }
   /**
@@ -173,10 +169,10 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * ```
    */
   addTools(tools) {
-    const transformedTools = transformTools(tools);
-    for (const tool of transformedTools) {
-      this.client.addTool(tool.openaiTool, tool.execute);
-    }
+    const openaiTools = transformTools(tools);
+    this.updateConfig({
+      tools: openaiTools.map((t) => t.openaiTool)
+    });
   }
   /**
    * Emits a speaking event using the configured voice model.
@@ -212,7 +208,7 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
     if (input.trim().length === 0) {
       throw new Error("Input text is empty");
     }
-    this.client.realtime.send("response.create", {
+    this.sendEvent("response.create", {
       response: {
         instructions: `Repeat the following text: ${input}`,
         voice: options?.speaker ? options.speaker : void 0
@@ -238,7 +234,7 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * ```
    */
   updateConfig(sessionConfig) {
-    this.client.updateSession(sessionConfig);
+    this.sendEvent("session.update", { session: sessionConfig });
   }
   /**
    * Processes audio input for speech recognition.
@@ -273,14 +269,14 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
       const buffer = Buffer.concat(chunks);
       const int16Array = new Int16Array(buffer.buffer, buffer.byteOffset ?? 0, (buffer.byteLength ?? 0) / 2);
       const base64Audio = this.int16ArrayToBase64(int16Array);
-      this.client.realtime.send("conversation.item.create", {
+      this.sendEvent("conversation.item.create", {
         item: {
           type: "message",
           role: "user",
           content: [{ type: "input_audio", audio: base64Audio }]
         }
       });
-      this.client.realtime.send("response.create", {
+      this.sendEvent("response.create", {
         response: {
           modalities: ["text"],
           instructions: `ONLY repeat the input and DO NOT say anything else`
@@ -290,6 +286,16 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
       this.emit("error", new Error("Unsupported audio data format"));
     }
   }
+  waitForOpen() {
+    return new Promise((resolve) => {
+      this.ws.on("open", resolve);
+    });
+  }
+  waitForSessionCreated() {
+    return new Promise((resolve) => {
+      this.client.on("session.created", resolve);
+    });
+  }
   /**
    * Establishes a connection to the OpenAI realtime service.
    * Must be called before using speak, listen, or relay functions.
@@ -303,8 +309,17 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * ```
    */
   async connect() {
-    await this.client.connect();
-    await this.client.waitForSessionCreated();
+    await this.waitForOpen();
+    await this.waitForSessionCreated();
+    const openaiTools = transformTools(this.tools);
+    this.updateConfig({
+      instructions: this.instructions,
+      tools: openaiTools.map((t) => t.openaiTool),
+      input_audio_transcription: {
+        model: "whisper-1"
+      },
+      voice: this.speaker
+    });
     this.state = "open";
   }
   /**
@@ -325,7 +340,7 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * await voice.relay(micStream);
    * ```
    */
-  async send(audioData) {
+  async send(audioData, eventId) {
     if (!this.state || this.state !== "open") {
       console.warn("Cannot relay audio when not open. Call open() first.");
       return;
@@ -335,15 +350,14 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
       stream.on("data", (chunk) => {
         try {
           const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-          const int16Array = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 2);
-          this.client.appendInputAudio(int16Array);
+          this.sendEvent("input_audio_buffer.append", { audio: buffer.toString("base64"), event_id: eventId });
         } catch (err) {
           this.emit("error", err);
         }
       });
     } else if (audioData instanceof Int16Array) {
       try {
-        this.client.appendInputAudio(audioData);
+        this.sendEvent("input_audio_buffer.append", { audio: audioData, event_id: eventId });
       } catch (err) {
         this.emit("error", err);
       }
@@ -370,7 +384,7 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
    * });
    */
   async answer({ options }) {
-    this.client.realtime.send("response.create", { response: options ?? {} });
+    this.sendEvent("response.create", { response: options ?? {} });
   }
   /**
    * Registers an event listener for voice events.
@@ -439,29 +453,105 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
     }
   }
   setupEventListeners() {
-    this.client.on("error", (error) => {
-      this.emit("error", error);
+    const speakerStreams = /* @__PURE__ */ new Map();
+    this.ws.on("message", (message) => {
+      const data = JSON.parse(message.toString());
+      this.client.emit(data.type, data);
+      if (this.debug) {
+        const { delta, ...fields } = data;
+        console.log(data.type, fields, delta?.length < 100 ? delta : "");
+      }
     });
-    this.client.on("conversation.created", (conversation) => {
-      this.emit("openAIRealtime:conversation.created", conversation);
+    this.client.on("session.created", (ev) => {
+      this.emit("session.created", ev);
     });
-    this.client.on("conversation.interrupted", () => {
-      this.emit("openAIRealtime:conversation.interrupted");
+    this.client.on("session.updated", (ev) => {
+      this.emit("session.updated", ev);
     });
-    this.client.on("conversation.updated", ({ delta }) => {
-      if (delta?.audio) {
-        this.emit("speaking", { audio: delta.audio });
-      }
+    this.client.on("response.created", (ev) => {
+      this.emit("response.created", ev);
+      const speakerStream = new stream.PassThrough();
+      speakerStream.id = ev.response.id;
+      speakerStreams.set(ev.response.id, speakerStream);
+      this.emit("speaker", speakerStream);
     });
-    this.client.on("conversation.item.appended", (item) => {
-      this.emit("openAIRealtime:conversation.item.appended", item);
+    this.client.on("conversation.item.input_audio_transcription.delta", (ev) => {
+      this.emit("transcribing", { text: ev.delta, response_id: ev.response_id, role: "user" });
     });
-    this.client.on("conversation.item.completed", ({ item, delta }) => {
-      if (item.formatted.transcript) {
-        this.emit("writing", { text: item.formatted.transcript, role: item.role });
-      }
-      this.emit("openAIRealtime:conversation.item.completed", { item, delta });
+    this.client.on("conversation.item.input_audio_transcription.done", (ev) => {
+      this.emit("transcribing", { text: "\n", response_id: ev.response_id, role: "user" });
+    });
+    this.client.on("response.audio.delta", (ev) => {
+      const audio = Buffer.from(ev.delta, "base64");
+      this.emit("speaking", { audio, response_id: ev.response_id });
+      const stream = speakerStreams.get(ev.response_id);
+      stream?.write(audio);
+    });
+    this.client.on("response.audio.done", (ev) => {
+      this.emit("speaking.done", { response_id: ev.response_id });
+      const stream = speakerStreams.get(ev.response_id);
+      stream?.end();
+    });
+    this.client.on("response.audio_transcript.delta", (ev) => {
+      this.emit("writing", { text: ev.delta, response_id: ev.response_id });
     });
+    this.client.on("response.audio_transcript.done", (ev) => {
+      this.emit("writing", { text: "\n", response_id: ev.response_id });
+    });
+    this.client.on("response.text.delta", (ev) => {
+      this.emit("writing", { text: ev.delta, response_id: ev.response_id });
+    });
+    this.client.on("response.text.done", (ev) => {
+      this.emit("writing", { text: "\n", response_id: ev.response_id });
+    });
+    this.client.on("response.done", async (ev) => {
+      await this.handleFunctionCalls(ev);
+      this.emit("response.done", ev);
+      speakerStreams.delete(ev.response.id);
+    });
+  }
+  async handleFunctionCalls(ev) {
+    for (const output of ev.response?.output ?? []) {
+      if (output.type === "function_call") {
+        await this.handleFunctionCall(output);
+      }
+    }
+  }
+  async handleFunctionCall(output) {
+    try {
+      const context = JSON.parse(output.arguments);
+      const tool = this.tools?.[output.name];
+      if (!tool) {
+        console.warn(`Tool "${output.name}" not found`);
+        return;
+      }
+      const result = await tool?.execute?.(
+        { context },
+        {
+          toolCallId: "unknown",
+          messages: []
+        }
+      );
+      this.sendEvent("conversation.item.create", {
+        item: {
+          type: "function_call_output",
+          call_id: output.call_id,
+          output: JSON.stringify(result)
+        }
+      });
+    } catch (e) {
+      const err = e;
+      console.warn(`Error calling tool "${output.name}":`, err.message);
+      this.sendEvent("conversation.item.create", {
+        item: {
+          type: "function_call_output",
+          call_id: output.call_id,
+          output: JSON.stringify({ error: err.message })
+        }
+      });
+    } finally {
+      this.sendEvent("response.create", {});
+    }
   }
   int16ArrayToBase64(int16Array) {
     const buffer = new ArrayBuffer(int16Array.length * 2);
@@ -476,6 +566,14 @@ var OpenAIRealtimeVoice = class extends voice.MastraVoice {
     }
     return btoa(binary);
   }
+  sendEvent(type, data) {
+    this.ws.send(
+      JSON.stringify({
+        type,
+        ...data
+      })
+    );
+  }
 };
 
 exports.OpenAIRealtimeVoice = OpenAIRealtimeVoice;