@orka-js/realtime 1.5.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Orka Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.cjs

@@ -0,0 +1,231 @@
+'use strict';
+
+var agent = require('@orka-js/agent');
+
+// src/realtime-agent.ts
+var RealtimeAgent = class {
+  config;
+  stt;
+  tts;
+  innerAgent;
+  constructor(options) {
+    this.config = options.config;
+    this.stt = options.stt;
+    this.tts = options.tts;
+    this.innerAgent = new agent.StreamingToolAgent(
+      {
+        goal: options.config.goal,
+        systemPrompt: options.config.systemPrompt,
+        tools: options.tools ?? []
+      },
+      options.llm
+    );
+  }
+  /**
+   * Process audio: transcribe → run LLM → synthesize.
+   * Returns final transcript, response text and audio buffer.
+   */
+  async process(audio, audioFormat = "audio/wav") {
+    const transcript = await this.stt.transcribe(audio, audioFormat);
+    let response = "";
+    for await (const event of this.innerAgent.runStream(transcript)) {
+      if (event.type === "done") {
+        response = event.content;
+      }
+    }
+    let audioOutput;
+    if (this.config.tts !== false && this.tts && response) {
+      audioOutput = await this.tts.synthesize(response);
+    }
+    return { transcript, response, audio: audioOutput };
+  }
+  /**
+   * Process audio as a stream of events — yields transcript, tokens, audio chunks, and done.
+   */
+  async *processStream(audio, audioFormat = "audio/wav") {
+    let transcript;
+    try {
+      transcript = await this.stt.transcribe(audio, audioFormat);
+    } catch (error) {
+      yield { type: "error", error, message: error.message };
+      return;
+    }
+    yield { type: "transcript", text: transcript };
+    let fullResponse = "";
+    for await (const event of this.innerAgent.runStream(transcript)) {
+      switch (event.type) {
+        case "token":
+          fullResponse += event.token;
+          yield { type: "token", content: event.token };
+          break;
+        case "tool_call":
+          yield { type: "tool_call", name: event.name, args: event.arguments };
+          break;
+        case "tool_result":
+          yield { type: "tool_result", name: String(event.toolCallId), result: event.result };
+          break;
+        case "done":
+          fullResponse = event.content || fullResponse;
+          break;
+        case "error":
+          yield { type: "error", error: event.error, message: event.error.message };
+          return;
+      }
+    }
+    let audioOutput;
+    if (this.config.tts !== false && this.tts && fullResponse) {
+      if (this.tts.synthesizeStream) {
+        for await (const chunk of this.tts.synthesizeStream(fullResponse)) {
+          yield { type: "audio_chunk", data: chunk };
+          if (!audioOutput) audioOutput = chunk;
+          else audioOutput = Buffer.concat([audioOutput, chunk]);
+        }
+      } else {
+        audioOutput = await this.tts.synthesize(fullResponse);
+        yield { type: "audio_chunk", data: audioOutput };
+      }
+    }
+    yield { type: "done", transcript, response: fullResponse, audio: audioOutput };
+  }
+  /**
+   * Returns an Express/Node.js-compatible WebSocket message handler.
+   * Expects binary audio messages from the client, sends back JSON events.
+   *
+   * @example
+   * ```typescript
+   * import { WebSocketServer } from 'ws';
+   * const wss = new WebSocketServer({ port: 8080 });
+   * wss.on('connection', agent.wsHandler());
+   * ```
+   */
+  wsHandler() {
+    return (ws) => {
+      ws.on("message", async (data) => {
+        const audio = data instanceof Buffer ? data : data instanceof ArrayBuffer ? Buffer.from(data) : Buffer.from(data);
+        try {
+          for await (const event of this.processStream(audio)) {
+            ws.send(JSON.stringify(event));
+          }
+        } catch (error) {
+          ws.send(JSON.stringify({
+            type: "error",
+            error: { message: error.message },
+            message: error.message
+          }));
+        }
+      });
+    };
+  }
+};
+
+// src/adapters/openai-stt.ts
+var OpenAISTTAdapter = class {
+  apiKey;
+  model;
+  baseURL;
+  timeoutMs;
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.model = config.model ?? "whisper-1";
+    this.baseURL = config.baseURL ?? "https://api.openai.com/v1";
+    this.timeoutMs = config.timeoutMs ?? 12e4;
+  }
+  async transcribe(audio, format = "audio/wav") {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+    const ext = format.includes("webm") ? "webm" : format.includes("mp3") ? "mp3" : format.includes("ogg") ? "ogg" : "wav";
+    const audioData = audio instanceof Buffer ? audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength) : audio;
+    const blob = new Blob([audioData], { type: format });
+    const formData = new FormData();
+    formData.append("file", blob, `audio.${ext}`);
+    formData.append("model", this.model);
+    formData.append("response_format", "text");
+    let response;
+    try {
+      response = await fetch(`${this.baseURL}/audio/transcriptions`, {
+        method: "POST",
+        signal: controller.signal,
+        headers: { "Authorization": `Bearer ${this.apiKey}` },
+        body: formData
+      });
+    } catch (error) {
+      clearTimeout(timeout);
+      if (error.name === "AbortError") {
+        throw new Error(`Whisper API request timed out after ${this.timeoutMs}ms`);
+      }
+      throw error;
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!response.ok) {
+      const err = await response.text();
+      throw new Error(`Whisper API error: ${response.status} - ${err}`);
+    }
+    return response.text();
+  }
+};
+
+// src/adapters/openai-tts.ts
+var OpenAITTSAdapter = class {
+  apiKey;
+  model;
+  defaultVoice;
+  baseURL;
+  timeoutMs;
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.model = config.model ?? "tts-1";
+    this.defaultVoice = config.voice ?? "alloy";
+    this.baseURL = config.baseURL ?? "https://api.openai.com/v1";
+    this.timeoutMs = config.timeoutMs ?? 6e4;
+  }
+  async synthesize(text, options = {}) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+    let response;
+    try {
+      response = await fetch(`${this.baseURL}/audio/speech`, {
+        method: "POST",
+        signal: controller.signal,
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({
+          model: this.model,
+          input: text,
+          voice: options.voice ?? this.defaultVoice,
+          response_format: options.format ?? "mp3",
+          speed: options.speed ?? 1
+        })
+      });
+    } catch (error) {
+      clearTimeout(timeout);
+      if (error.name === "AbortError") {
+        throw new Error(`TTS API request timed out after ${this.timeoutMs}ms`);
+      }
+      throw error;
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!response.ok) {
+      const err = await response.text();
+      throw new Error(`TTS API error: ${response.status} - ${err}`);
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    return Buffer.from(arrayBuffer);
+  }
+  async *synthesizeStream(text, options = {}) {
+    const sentences = text.match(/[^.!?]+[.!?]+/g) ?? [text];
+    for (const sentence of sentences) {
+      const audio = await this.synthesize(sentence.trim(), options);
+      yield audio;
+    }
+  }
+};
+
+exports.OpenAISTTAdapter = OpenAISTTAdapter;
+exports.OpenAITTSAdapter = OpenAITTSAdapter;
+exports.RealtimeAgent = RealtimeAgent;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/realtime-agent.ts","../src/adapters/openai-stt.ts","../src/adapters/openai-tts.ts"],"names":["StreamingToolAgent"],"mappings":";;;;;AAoCO,IAAM,gBAAN,MAAoB;AAAA,EACjB,MAAA;AAAA,EACA,GAAA;AAAA,EACA,GAAA;AAAA,EACA,UAAA;AAAA,EAER,YAAY,OAAA,EAA+B;AACzC,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,MAAM,OAAA,CAAQ,GAAA;AACnB,IAAA,IAAA,CAAK,MAAM,OAAA,CAAQ,GAAA;AAEnB,IAAA,IAAA,CAAK,aAAa,IAAIA,wBAAA;AAAA,MACpB;AAAA,QACE,IAAA,EAAM,QAAQ,MAAA,CAAO,IAAA;AAAA,QACrB,YAAA,EAAc,QAAQ,MAAA,CAAO,YAAA;AAAA,QAC7B,KAAA,EAAO,OAAA,CAAQ,KAAA,IAAS;AAAC,OAC3B;AAAA,MACA,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAA,CAAQ,KAAA,EAA6B,WAAA,GAAc,WAAA,EAA6C;AACpG,IAAA,MAAM,aAAa,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,OAAO,WAAW,CAAA;AAE/D,IAAA,IAAI,QAAA,GAAW,EAAA;AACf,IAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,UAAA,CAAW,SAAA,CAAU,UAAU,CAAA,EAAG;AAC/D,MAAA,IAAI,KAAA,CAAM,SAAS,MAAA,EAAQ;AACzB,QAAA,QAAA,GAAW,KAAA,CAAM,OAAA;AAAA,MACnB;AAAA,IACF;AAEA,IAAA,IAAI,WAAA;AACJ,IAAA,IAAI,KAAK,MAAA,CAAO,GAAA,KAAQ,KAAA,IAAS,IAAA,CAAK,OAAO,QAAA,EAAU;AACrD,MAAA,WAAA,GAAc,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,QAAQ,CAAA;AAAA,IAClD;AAEA,IAAA,OAAO,EAAE,UAAA,EAAY,QAAA,EAAU,KAAA,EAAO,WAAA,EAAY;AAAA,EACpD;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,aAAA,CACL,KAAA,EACA,WAAA,GAAc,WAAA,EACgB;AAC9B,IAAA,IAAI,UAAA;AACJ,IAAA,IAAI;AACF,MAAA,UAAA,GAAa,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,OAAO,WAAW,CAAA;AAAA,IAC3D,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,EAAE,IAAA,EAAM,OAAA,EAAS,KAAA,EAAuB,OAAA,EAAU,MAAgB,OAAA,EAAQ;AAChF,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,IAAA,EAAM,YAAA,EAAc,IAAA,EAAM,UAAA,EAAW;AAE7C,IAAA,IAAI,YAAA,GAAe,EAAA;AAEnB,IAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,UAAA,CAAW,SAAA,CAAU,UAAU,CAAA,EAAG;AAC/D,MAAA,QAAQ,MAAM,IAAA;AAAM,QAClB,KAAK,OAAA;AACH,UAAA,YAAA,IAAgB,KAAA,CAAM,KAAA;AACtB,UAAA,MAAM,EAAE,IAAA,EAAM,OAAA,EAAS,OAAA,EAAS,MAAM,KAAA,EAAM;AAC5C,UAAA;AAAA,QACF,KAAK,WAAA;AACH,UAAA,MAAM,EAAE,MAAM,WAAA,EAAa,IAAA,EAAM,MAAM,IAAA,EAAM,IAAA,EAAM,MAAM,SAAA,EAAU;AACnE,UAAA;AAAA,QACF,KAAK,aAAA;AACH,UAAA,MAAM,EAAE,IAAA,EAAM,aAAA,EAAe,IAAA,EAAM,MAAA,CAAO,MAAM,UAAU,CAAA,EAAG,MAAA,EAAQ,KAAA,CAAM,MAAA,EAAO;AAClF,UAAA;AAAA,QACF,KAAK,MAAA;AACH,UAAA,YAAA,GAAe,MAAM,OAAA,IAAW,YAAA;AAChC,UAAA;AAAA,QACF,KAAK,OAAA;AACH,UAAA,MAAM,EAAE,MAAM,OAAA,EAAS,KAAA,EAAO,MAAM,KAAA,EAAO,OAAA,EAAS,KAAA,CAAM,KAAA,CAAM,OAAA,EAAQ;AACxE,UAAA;AAAA;AACJ,IACF;AAGA,IAAA,IAAI,WAAA;AACJ,IAAA,IAAI,KAAK,MAAA,CAAO,GAAA,KAAQ,KAAA,IAAS,IAAA,CAAK,OAAO,YAAA,EAAc;AACzD,MAAA,IAAI,IAAA,CAAK,IAAI,gBAAA,EAAkB;AAC7B,QAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,GAAA,CAAI,gBAAA,CAAiB,YAAY,CAAA,EAAG;AACjE,UAAA,MAAM,EAAE,IAAA,EAAM,aAAA,EAAe,IAAA,EAAM,KAAA,EAAM;AACzC,UAAA,IAAI,CAAC,aAAa,WAAA,GAAc,KAAA;AAAA,6BACb,MAAA,CAAO,MAAA,CAAO,CAAC,WAAA,EAAa,KAAK,CAAC,CAAA;AAAA,QACvD;AAAA,MACF,CAAA,MAAO;AACL,QAAA,WAAA,GAAc,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,YAAY,CAAA;AACpD,QAAA,MAAM,EAAE,IAAA,EAAM,aAAA,EAAe,IAAA,EAAM,WAAA,EAAY;AAAA,MACjD;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,IAAA,EAAM,MAAA,EAAQ,YAAY,QAAA,EAAU,YAAA,EAAc,OAAO,WAAA,EAAY;AAAA,EAC/E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,SAAA,GAAY;AACV,IAAA,OAAO,CAAC,EAAA,KAGF;AACJ,MAAA,EAAA,CAAG,EAAA,CAAG,SAAA,EAAW,OAAO,IAAA,KAAkB;AACxC,QAAA,MAAM,KAAA,GAAQ,IAAA,YAAgB,MAAA,GAAS,IAAA,GACnC,IAAA,YAAgB,WAAA,GAAc,MAAA,CAAO,IAAA,CAAK,IAAI,CAAA,GAC9C,MAAA,CAAO,IAAA,CAAK,IAAkB,CAAA;AAElC,QAAA,IAAI;AACF,UAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA,EAAG;AACnD,YAAA,EAAA,CAAG,IAAA,CAAK,IAAA,CAAK,SAAA,CAAU,KAAK,CAAC,CAAA;AAAA,UAC/B;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,EAAA,CAAG,IAAA,CAAK,KAAK,SAAA,CAAU;AAAA,YACrB,IAAA,EAAM,OAAA;AAAA,YACN,KAAA,EAAO,EAAE,OAAA,EAAU,KAAA,CAAgB,OAAA,EAAQ;AAAA,YAC3C,SAAU,KAAA,CAAgB;AAAA,WAC3B,CAAC,CAAA;AAAA,QACJ;AAAA,MACF,CAAC,CAAA;AAAA,IACH,CAAA;AAAA,EACF;AACF;;;AChKO,IAAM,mBAAN,MAA6C;AAAA,EAC1C,MAAA;AAAA,EACA,KAAA;AAAA,EACA,OAAA;AAAA,EACA,SAAA;AAAA,EAER,YAAY,MAAA,EAAyB;AACnC,IAAA,IAAA,CAAK,SAAS,MAAA,CAAO,MAAA;AACrB,IAAA,IAAA,CAAK,KAAA,GAAQ,OAAO,KAAA,IAAS,WAAA;AAC7B,IAAA,IAAA,CAAK,OAAA,GAAU,OAAO,OAAA,IAAW,2BAAA;AACjC,IAAA,IAAA,CAAK,SAAA,GAAY,OAAO,SAAA,IAAa,IAAA;AAAA,EACvC;AAAA,EAEA,MAAM,UAAA,CAAW,KAAA,EAA6B,MAAA,GAAS,WAAA,EAA8B;AACnF,IAAA,MAAM,UAAA,GAAa,IAAI,eAAA,EAAgB;AACvC,IAAA,MAAM,UAAU,UAAA,CAAW,MAAM,WAAW,KAAA,EAAM,EAAG,KAAK,SAAS,CAAA;AAEnE,IAAA,MAAM,GAAA,GAAM,MAAA,CAAO,QAAA,CAAS,MAAM,IAAI,MAAA,GAClC,MAAA,CAAO,QAAA,CAAS,KAAK,IAAI,KAAA,GACzB,MAAA,CAAO,QAAA,CAAS,KAAK,IAAI,KAAA,GACzB,KAAA;AAEJ,IAAA,MAAM,SAAA,GAAY,KAAA,YAAiB,MAAA,GAAS,KAAA,CAAM,MAAA,CAAO,KAAA,CAAM,KAAA,CAAM,UAAA,EAAY,KAAA,CAAM,UAAA,GAAa,KAAA,CAAM,UAAU,CAAA,GAAI,KAAA;AACxH,IAAA,MAAM,IAAA,GAAO,IAAI,IAAA,CAAK,CAAC,SAAqB,CAAA,EAAG,EAAE,IAAA,EAAM,MAAA,EAAQ,CAAA;AAC/D,IAAA,MAAM,QAAA,GAAW,IAAI,QAAA,EAAS;AAC9B,IAAA,QAAA,CAAS,MAAA,CAAO,MAAA,EAAQ,IAAA,EAAM,CAAA,MAAA,EAAS,GAAG,CAAA,CAAE,CAAA;AAC5C,IAAA,QAAA,CAAS,MAAA,CAAO,OAAA,EAAS,IAAA,CAAK,KAAK,CAAA;AACnC,IAAA,QAAA,CAAS,MAAA,CAAO,mBAAmB,MAAM,CAAA;AAEzC,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,OAAO,CAAA,qBAAA,CAAA,EAAyB;AAAA,QAC7D,MAAA,EAAQ,MAAA;AAAA,QACR,QAAQ,UAAA,CAAW,MAAA;AAAA,QACnB,SAAS,EAAE,eAAA,EAAiB,CAAA,OAAA,EAAU,IAAA,CAAK,MAAM,CAAA,CAAA,EAAG;AAAA,QACpD,IAAA,EAAM;AAAA,OACP,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,IAAK,KAAA,CAAgB,SAAS,YAAA,EAAc;AAC1C,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,oCAAA,EAAuC,IAAA,CAAK,SAAS,CAAA,EAAA,CAAI,CAAA;AAAA,MAC3E;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,YAAA,CAAa,OAAO,CAAA;AAAA,IACtB;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,GAAA,GAAM,MAAM,QAAA,CAAS,IAAA,EAAK;AAChC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mBAAA,EAAsB,SAAS,MAAM,CAAA,GAAA,EAAM,GAAG,CAAA,CAAE,CAAA;AAAA,IAClE;AAEA,IAAA,OAAO,SAAS,IAAA,EAAK;AAAA,EACvB;AACF;;;ACrDO,IAAM,mBAAN,MAA6C;AAAA,EAC1C,MAAA;AAAA,EACA,KAAA;AAAA,EACA,YAAA;AAAA,EACA,OAAA;AAAA,EACA,SAAA;AAAA,EAER,YAAY,MAAA,EAAyB;AACnC,IAAA,IAAA,CAAK,SAAS,MAAA,CAAO,MAAA;AACrB,IAAA,IAAA,CAAK,KAAA,GAAQ,OAAO,KAAA,IAAS,OAAA;AAC7B,IAAA,IAAA,CAAK,YAAA,GAAe,OAAO,KAAA,IAAS,OAAA;AACpC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAO,OAAA,IAAW,2BAAA;AACjC,IAAA,IAAA,CAAK,SAAA,GAAY,OAAO,SAAA,IAAa,GAAA;AAAA,EACvC;AAAA,EAEA,MAAM,UAAA,CAAW,IAAA,EAAc,OAAA,GAAgC,EAAC,EAAoB;AAClF,IAAA,MAAM,UAAA,GAAa,IAAI,eAAA,EAAgB;AACvC,IAAA,MAAM,UAAU,UAAA,CAAW,MAAM,WAAW,KAAA,EAAM,EAAG,KAAK,SAAS,CAAA;AAEnE,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,OAAO,CAAA,aAAA,CAAA,EAAiB;AAAA,QACrD,MAAA,EAAQ,MAAA;AAAA,QACR,QAAQ,UAAA,CAAW,MAAA;AAAA,QACnB,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB,kBAAA;AAAA,UAChB,eAAA,EAAiB,CAAA,OAAA,EAAU,IAAA,CAAK,MAAM,CAAA;AAAA,SACxC;AAAA,QACA,IAAA,EAAM,KAAK,SAAA,CAAU;AAAA,UACnB,OAAO,IAAA,CAAK,KAAA;AAAA,UACZ,KAAA,EAAO,IAAA;AAAA,UACP,KAAA,EAAO,OAAA,CAAQ,KAAA,IAAS,IAAA,CAAK,YAAA;AAAA,UAC7B,eAAA,EAAiB,QAAQ,MAAA,IAAU,KAAA;AAAA,UACnC,KAAA,EAAO,QAAQ,KAAA,IAAS;AAAA,SACzB;AAAA,OACF,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,IAAK,KAAA,CAAgB,SAAS,YAAA,EAAc;AAC1C,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,gCAAA,EAAmC,IAAA,CAAK,SAAS,CAAA,EAAA,CAAI,CAAA;AAAA,MACvE;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,YAAA,CAAa,OAAO,CAAA;AAAA,IACtB;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,GAAA,GAAM,MAAM,QAAA,CAAS,IAAA,EAAK;AAChC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,eAAA,EAAkB,SAAS,MAAM,CAAA,GAAA,EAAM,GAAG,CAAA,CAAE,CAAA;AAAA,IAC9D;AAEA,IAAA,MAAM,WAAA,GAAc,MAAM,QAAA,CAAS,WAAA,EAAY;AAC/C,IAAA,OAAO,MAAA,CAAO,KAAK,WAAW,CAAA;AAAA,EAChC;AAAA,EAEA,OAAO,gBAAA,CAAiB,IAAA,EAAc,OAAA,GAAgC,EAAC,EAA0B;AAE/F,IAAA,MAAM,YAAY,IAAA,CAAK,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,IAAI,CAAA;AACvD,IAAA,KAAA,MAAW,YAAY,SAAA,EAAW;AAChC,MAAA,MAAM,QAAQ,MAAM,IAAA,CAAK,WAAW,QAAA,CAAS,IAAA,IAAQ,OAAO,CAAA;AAC5D,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF","file":"index.cjs","sourcesContent":["import type { LLMAdapter } from '@orka-js/core';\nimport { StreamingToolAgent } from '@orka-js/agent';\nimport type { Tool } from '@orka-js/agent';\nimport type {\n  STTAdapter,\n  TTSAdapter,\n  RealtimeAgentConfig,\n  RealtimeEvent,\n  RealtimeProcessResult,\n} from './types.js';\n\nexport interface RealtimeAgentOptions {\n  config: RealtimeAgentConfig;\n  llm: LLMAdapter;\n  stt: STTAdapter;\n  tts?: TTSAdapter;\n  tools?: Tool[];\n}\n\n/**\n * A voice agent that processes audio through the STT → LLM → TTS pipeline.\n *\n * @example\n * ```typescript\n * const agent = new RealtimeAgent({\n *   config: { goal: 'Answer questions', tts: true },\n *   llm: new OpenAIAdapter({ apiKey }),\n *   stt: new OpenAISTTAdapter({ apiKey }),\n *   tts: new OpenAITTSAdapter({ apiKey }),\n * });\n *\n * const result = await agent.process(audioBuffer);\n * console.log(result.transcript); // what the user said\n * console.log(result.response);   // the LLM's reply\n * ```\n */\nexport class RealtimeAgent {\n  private config: RealtimeAgentConfig;\n  private stt: STTAdapter;\n  private tts?: TTSAdapter;\n  private innerAgent: StreamingToolAgent;\n\n  constructor(options: RealtimeAgentOptions) {\n    this.config = options.config;\n    this.stt = options.stt;\n    this.tts = options.tts;\n\n    this.innerAgent = new StreamingToolAgent(\n      {\n        goal: options.config.goal,\n        systemPrompt: options.config.systemPrompt,\n        tools: options.tools ?? [],\n      },\n      options.llm,\n    );\n  }\n\n  /**\n   * Process audio: transcribe → run LLM → synthesize.\n   * Returns final transcript, response text and audio buffer.\n   */\n  async process(audio: Buffer | ArrayBuffer, audioFormat = 'audio/wav'): Promise<RealtimeProcessResult> {\n    const transcript = await this.stt.transcribe(audio, audioFormat);\n\n    let response = '';\n    for await (const event of this.innerAgent.runStream(transcript)) {\n      if (event.type === 'done') {\n        response = event.content;\n      }\n    }\n\n    let audioOutput: Buffer | undefined;\n    if (this.config.tts !== false && this.tts && response) {\n      audioOutput = await this.tts.synthesize(response);\n    }\n\n    return { transcript, response, audio: audioOutput };\n  }\n\n  /**\n   * Process audio as a stream of events — yields transcript, tokens, audio chunks, and done.\n   */\n  async *processStream(\n    audio: Buffer | ArrayBuffer,\n    audioFormat = 'audio/wav',\n  ): AsyncIterable<RealtimeEvent> {\n    let transcript: string;\n    try {\n      transcript = await this.stt.transcribe(audio, audioFormat);\n    } catch (error) {\n      yield { type: 'error', error: error as Error, message: (error as Error).message };\n      return;\n    }\n\n    yield { type: 'transcript', text: transcript };\n\n    let fullResponse = '';\n\n    for await (const event of this.innerAgent.runStream(transcript)) {\n      switch (event.type) {\n        case 'token':\n          fullResponse += event.token;\n          yield { type: 'token', content: event.token };\n          break;\n        case 'tool_call':\n          yield { type: 'tool_call', name: event.name, args: event.arguments };\n          break;\n        case 'tool_result':\n          yield { type: 'tool_result', name: String(event.toolCallId), result: event.result };\n          break;\n        case 'done':\n          fullResponse = event.content || fullResponse;\n          break;\n        case 'error':\n          yield { type: 'error', error: event.error, message: event.error.message };\n          return;\n      }\n    }\n\n    // Synthesize audio\n    let audioOutput: Buffer | undefined;\n    if (this.config.tts !== false && this.tts && fullResponse) {\n      if (this.tts.synthesizeStream) {\n        for await (const chunk of this.tts.synthesizeStream(fullResponse)) {\n          yield { type: 'audio_chunk', data: chunk };\n          if (!audioOutput) audioOutput = chunk;\n          else audioOutput = Buffer.concat([audioOutput, chunk]);\n        }\n      } else {\n        audioOutput = await this.tts.synthesize(fullResponse);\n        yield { type: 'audio_chunk', data: audioOutput };\n      }\n    }\n\n    yield { type: 'done', transcript, response: fullResponse, audio: audioOutput };\n  }\n\n  /**\n   * Returns an Express/Node.js-compatible WebSocket message handler.\n   * Expects binary audio messages from the client, sends back JSON events.\n   *\n   * @example\n   * ```typescript\n   * import { WebSocketServer } from 'ws';\n   * const wss = new WebSocketServer({ port: 8080 });\n   * wss.on('connection', agent.wsHandler());\n   * ```\n   */\n  wsHandler() {\n    return (ws: {\n      on(event: string, listener: (...args: unknown[]) => void): void;\n      send(data: string): void;\n    }) => {\n      ws.on('message', async (data: unknown) => {\n        const audio = data instanceof Buffer ? data\n          : data instanceof ArrayBuffer ? Buffer.from(data)\n          : Buffer.from(data as Uint8Array);\n\n        try {\n          for await (const event of this.processStream(audio)) {\n            ws.send(JSON.stringify(event));\n          }\n        } catch (error) {\n          ws.send(JSON.stringify({\n            type: 'error',\n            error: { message: (error as Error).message },\n            message: (error as Error).message,\n          }));\n        }\n      });\n    };\n  }\n}\n","import type { STTAdapter } from '../types.js';\n\nexport interface OpenAISTTConfig {\n  apiKey: string;\n  model?: string;\n  baseURL?: string;\n  timeoutMs?: number;\n}\n\n/**\n * STT adapter using OpenAI Whisper API.\n */\nexport class OpenAISTTAdapter implements STTAdapter {\n  private apiKey: string;\n  private model: string;\n  private baseURL: string;\n  private timeoutMs: number;\n\n  constructor(config: OpenAISTTConfig) {\n    this.apiKey = config.apiKey;\n    this.model = config.model ?? 'whisper-1';\n    this.baseURL = config.baseURL ?? 'https://api.openai.com/v1';\n    this.timeoutMs = config.timeoutMs ?? 120_000;\n  }\n\n  async transcribe(audio: Buffer | ArrayBuffer, format = 'audio/wav'): Promise<string> {\n    const controller = new AbortController();\n    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);\n\n    const ext = format.includes('webm') ? 'webm'\n      : format.includes('mp3') ? 'mp3'\n      : format.includes('ogg') ? 'ogg'\n      : 'wav';\n\n    const audioData = audio instanceof Buffer ? audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength) : audio;\n    const blob = new Blob([audioData as BlobPart], { type: format });\n    const formData = new FormData();\n    formData.append('file', blob, `audio.${ext}`);\n    formData.append('model', this.model);\n    formData.append('response_format', 'text');\n\n    let response: Response;\n    try {\n      response = await fetch(`${this.baseURL}/audio/transcriptions`, {\n        method: 'POST',\n        signal: controller.signal,\n        headers: { 'Authorization': `Bearer ${this.apiKey}` },\n        body: formData,\n      });\n    } catch (error) {\n      clearTimeout(timeout);\n      if ((error as Error).name === 'AbortError') {\n        throw new Error(`Whisper API request timed out after ${this.timeoutMs}ms`);\n      }\n      throw error;\n    } finally {\n      clearTimeout(timeout);\n    }\n\n    if (!response.ok) {\n      const err = await response.text();\n      throw new Error(`Whisper API error: ${response.status} - ${err}`);\n    }\n\n    return response.text();\n  }\n}\n","import type { TTSAdapter, TTSSynthesizeOptions } from '../types.js';\n\nexport interface OpenAITTSConfig {\n  apiKey: string;\n  model?: string;\n  voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';\n  baseURL?: string;\n  timeoutMs?: number;\n}\n\n/**\n * TTS adapter using OpenAI TTS API.\n */\nexport class OpenAITTSAdapter implements TTSAdapter {\n  private apiKey: string;\n  private model: string;\n  private defaultVoice: string;\n  private baseURL: string;\n  private timeoutMs: number;\n\n  constructor(config: OpenAITTSConfig) {\n    this.apiKey = config.apiKey;\n    this.model = config.model ?? 'tts-1';\n    this.defaultVoice = config.voice ?? 'alloy';\n    this.baseURL = config.baseURL ?? 'https://api.openai.com/v1';\n    this.timeoutMs = config.timeoutMs ?? 60_000;\n  }\n\n  async synthesize(text: string, options: TTSSynthesizeOptions = {}): Promise<Buffer> {\n    const controller = new AbortController();\n    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);\n\n    let response: Response;\n    try {\n      response = await fetch(`${this.baseURL}/audio/speech`, {\n        method: 'POST',\n        signal: controller.signal,\n        headers: {\n          'Content-Type': 'application/json',\n          'Authorization': `Bearer ${this.apiKey}`,\n        },\n        body: JSON.stringify({\n          model: this.model,\n          input: text,\n          voice: options.voice ?? this.defaultVoice,\n          response_format: options.format ?? 'mp3',\n          speed: options.speed ?? 1.0,\n        }),\n      });\n    } catch (error) {\n      clearTimeout(timeout);\n      if ((error as Error).name === 'AbortError') {\n        throw new Error(`TTS API request timed out after ${this.timeoutMs}ms`);\n      }\n      throw error;\n    } finally {\n      clearTimeout(timeout);\n    }\n\n    if (!response.ok) {\n      const err = await response.text();\n      throw new Error(`TTS API error: ${response.status} - ${err}`);\n    }\n\n    const arrayBuffer = await response.arrayBuffer();\n    return Buffer.from(arrayBuffer);\n  }\n\n  async *synthesizeStream(text: string, options: TTSSynthesizeOptions = {}): AsyncIterable<Buffer> {\n    // Split text into sentences for lower latency\n    const sentences = text.match(/[^.!?]+[.!?]+/g) ?? [text];\n    for (const sentence of sentences) {\n      const audio = await this.synthesize(sentence.trim(), options);\n      yield audio;\n    }\n  }\n}\n"]}

package/dist/index.d.cts

@@ -0,0 +1,178 @@
+import { LLMAdapter } from '@orka-js/core';
+import { Tool } from '@orka-js/agent';
+
+/**
+ * Adapter for Speech-to-Text (audio transcription).
+ */
+interface STTAdapter {
+    /**
+     * Transcribe audio buffer to text.
+     * @param audio Raw audio bytes
+     * @param format Audio MIME type (e.g. 'audio/webm', 'audio/wav', 'audio/mp3')
+     */
+    transcribe(audio: Buffer | ArrayBuffer, format?: string): Promise<string>;
+}
+/**
+ * Adapter for Text-to-Speech (audio synthesis).
+ */
+interface TTSAdapter {
+    /**
+     * Synthesize text to audio buffer.
+     */
+    synthesize(text: string, options?: TTSSynthesizeOptions): Promise<Buffer>;
+    /**
+     * Synthesize text to audio as a stream of chunks (optional).
+     */
+    synthesizeStream?(text: string, options?: TTSSynthesizeOptions): AsyncIterable<Buffer>;
+}
+interface TTSSynthesizeOptions {
+    voice?: string;
+    speed?: number;
+    /** Output format: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' */
+    format?: string;
+}
+/**
+ * Events emitted by RealtimeAgent during a voice interaction.
+ */
+type RealtimeEvent = {
+    type: 'transcript';
+    text: string;
+} | {
+    type: 'token';
+    content: string;
+} | {
+    type: 'tool_call';
+    name: string;
+    args: unknown;
+} | {
+    type: 'tool_result';
+    name: string;
+    result: unknown;
+} | {
+    type: 'audio_chunk';
+    data: Buffer;
+} | {
+    type: 'done';
+    transcript: string;
+    response: string;
+    audio?: Buffer;
+} | {
+    type: 'error';
+    error: Error;
+    message: string;
+};
+interface RealtimeAgentConfig {
+    /** System goal / personality for the voice agent */
+    goal: string;
+    /** Custom system prompt injected into the LLM */
+    systemPrompt?: string;
+    /** Language for STT (ISO-639-1, e.g. 'en', 'fr') */
+    language?: string;
+    /** Whether to synthesize audio output */
+    tts?: boolean;
+}
+interface RealtimeProcessResult {
+    /** Transcribed user speech */
+    transcript: string;
+    /** LLM text response */
+    response: string;
+    /** Synthesized audio (if tts is enabled) */
+    audio?: Buffer;
+}
+
+interface RealtimeAgentOptions {
+    config: RealtimeAgentConfig;
+    llm: LLMAdapter;
+    stt: STTAdapter;
+    tts?: TTSAdapter;
+    tools?: Tool[];
+}
+/**
+ * A voice agent that processes audio through the STT → LLM → TTS pipeline.
+ *
+ * @example
+ * ```typescript
+ * const agent = new RealtimeAgent({
+ *   config: { goal: 'Answer questions', tts: true },
+ *   llm: new OpenAIAdapter({ apiKey }),
+ *   stt: new OpenAISTTAdapter({ apiKey }),
+ *   tts: new OpenAITTSAdapter({ apiKey }),
+ * });
+ *
+ * const result = await agent.process(audioBuffer);
+ * console.log(result.transcript); // what the user said
+ * console.log(result.response);   // the LLM's reply
+ * ```
+ */
+declare class RealtimeAgent {
+    private config;
+    private stt;
+    private tts?;
+    private innerAgent;
+    constructor(options: RealtimeAgentOptions);
+    /**
+     * Process audio: transcribe → run LLM → synthesize.
+     * Returns final transcript, response text and audio buffer.
+     */
+    process(audio: Buffer | ArrayBuffer, audioFormat?: string): Promise<RealtimeProcessResult>;
+    /**
+     * Process audio as a stream of events — yields transcript, tokens, audio chunks, and done.
+     */
+    processStream(audio: Buffer | ArrayBuffer, audioFormat?: string): AsyncIterable<RealtimeEvent>;
+    /**
+     * Returns an Express/Node.js-compatible WebSocket message handler.
+     * Expects binary audio messages from the client, sends back JSON events.
+     *
+     * @example
+     * ```typescript
+     * import { WebSocketServer } from 'ws';
+     * const wss = new WebSocketServer({ port: 8080 });
+     * wss.on('connection', agent.wsHandler());
+     * ```
+     */
+    wsHandler(): (ws: {
+        on(event: string, listener: (...args: unknown[]) => void): void;
+        send(data: string): void;
+    }) => void;
+}
+
+interface OpenAISTTConfig {
+    apiKey: string;
+    model?: string;
+    baseURL?: string;
+    timeoutMs?: number;
+}
+/**
+ * STT adapter using OpenAI Whisper API.
+ */
+declare class OpenAISTTAdapter implements STTAdapter {
+    private apiKey;
+    private model;
+    private baseURL;
+    private timeoutMs;
+    constructor(config: OpenAISTTConfig);
+    transcribe(audio: Buffer | ArrayBuffer, format?: string): Promise<string>;
+}
+
+interface OpenAITTSConfig {
+    apiKey: string;
+    model?: string;
+    voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
+    baseURL?: string;
+    timeoutMs?: number;
+}
+/**
+ * TTS adapter using OpenAI TTS API.
+ */
+declare class OpenAITTSAdapter implements TTSAdapter {
+    private apiKey;
+    private model;
+    private defaultVoice;
+    private baseURL;
+    private timeoutMs;
+    constructor(config: OpenAITTSConfig);
+    synthesize(text: string, options?: TTSSynthesizeOptions): Promise<Buffer>;
+    synthesizeStream(text: string, options?: TTSSynthesizeOptions): AsyncIterable<Buffer>;
+}
+
+export { OpenAISTTAdapter, type OpenAISTTConfig, OpenAITTSAdapter, type OpenAITTSConfig, RealtimeAgent, type RealtimeAgentConfig, type RealtimeAgentOptions, type RealtimeEvent, type RealtimeProcessResult, type STTAdapter, type TTSAdapter, type TTSSynthesizeOptions };

package/dist/index.d.ts

@@ -0,0 +1,178 @@
+import { LLMAdapter } from '@orka-js/core';
+import { Tool } from '@orka-js/agent';
+
+/**
+ * Adapter for Speech-to-Text (audio transcription).
+ */
+interface STTAdapter {
+    /**
+     * Transcribe audio buffer to text.
+     * @param audio Raw audio bytes
+     * @param format Audio MIME type (e.g. 'audio/webm', 'audio/wav', 'audio/mp3')
+     */
+    transcribe(audio: Buffer | ArrayBuffer, format?: string): Promise<string>;
+}
+/**
+ * Adapter for Text-to-Speech (audio synthesis).
+ */
+interface TTSAdapter {
+    /**
+     * Synthesize text to audio buffer.
+     */
+    synthesize(text: string, options?: TTSSynthesizeOptions): Promise<Buffer>;
+    /**
+     * Synthesize text to audio as a stream of chunks (optional).
+     */
+    synthesizeStream?(text: string, options?: TTSSynthesizeOptions): AsyncIterable<Buffer>;
+}
+interface TTSSynthesizeOptions {
+    voice?: string;
+    speed?: number;
+    /** Output format: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' */
+    format?: string;
+}
+/**
+ * Events emitted by RealtimeAgent during a voice interaction.
+ */
+type RealtimeEvent = {
+    type: 'transcript';
+    text: string;
+} | {
+    type: 'token';
+    content: string;
+} | {
+    type: 'tool_call';
+    name: string;
+    args: unknown;
+} | {
+    type: 'tool_result';
+    name: string;
+    result: unknown;
+} | {
+    type: 'audio_chunk';
+    data: Buffer;
+} | {
+    type: 'done';
+    transcript: string;
+    response: string;
+    audio?: Buffer;
+} | {
+    type: 'error';
+    error: Error;
+    message: string;
+};
+interface RealtimeAgentConfig {
+    /** System goal / personality for the voice agent */
+    goal: string;
+    /** Custom system prompt injected into the LLM */
+    systemPrompt?: string;
+    /** Language for STT (ISO-639-1, e.g. 'en', 'fr') */
+    language?: string;
+    /** Whether to synthesize audio output */
+    tts?: boolean;
+}
+interface RealtimeProcessResult {
+    /** Transcribed user speech */
+    transcript: string;
+    /** LLM text response */
+    response: string;
+    /** Synthesized audio (if tts is enabled) */
+    audio?: Buffer;
+}
+
+interface RealtimeAgentOptions {
+    config: RealtimeAgentConfig;
+    llm: LLMAdapter;
+    stt: STTAdapter;
+    tts?: TTSAdapter;
+    tools?: Tool[];
+}
+/**
+ * A voice agent that processes audio through the STT → LLM → TTS pipeline.
+ *
+ * @example
+ * ```typescript
+ * const agent = new RealtimeAgent({
+ *   config: { goal: 'Answer questions', tts: true },
+ *   llm: new OpenAIAdapter({ apiKey }),
+ *   stt: new OpenAISTTAdapter({ apiKey }),
+ *   tts: new OpenAITTSAdapter({ apiKey }),
+ * });
+ *
+ * const result = await agent.process(audioBuffer);
+ * console.log(result.transcript); // what the user said
+ * console.log(result.response);   // the LLM's reply
+ * ```
+ */
+declare class RealtimeAgent {
+    private config;
+    private stt;
+    private tts?;
+    private innerAgent;
+    constructor(options: RealtimeAgentOptions);
+    /**
+     * Process audio: transcribe → run LLM → synthesize.
+     * Returns final transcript, response text and audio buffer.
+     */
+    process(audio: Buffer | ArrayBuffer, audioFormat?: string): Promise<RealtimeProcessResult>;
+    /**
+     * Process audio as a stream of events — yields transcript, tokens, audio chunks, and done.
+     */
+    processStream(audio: Buffer | ArrayBuffer, audioFormat?: string): AsyncIterable<RealtimeEvent>;
+    /**
+     * Returns an Express/Node.js-compatible WebSocket message handler.
+     * Expects binary audio messages from the client, sends back JSON events.
+     *
+     * @example
+     * ```typescript
+     * import { WebSocketServer } from 'ws';
+     * const wss = new WebSocketServer({ port: 8080 });
+     * wss.on('connection', agent.wsHandler());
+     * ```
+     */
+    wsHandler(): (ws: {
+        on(event: string, listener: (...args: unknown[]) => void): void;
+        send(data: string): void;
+    }) => void;
+}
+
+interface OpenAISTTConfig {
+    apiKey: string;
+    model?: string;
+    baseURL?: string;
+    timeoutMs?: number;
+}
+/**
+ * STT adapter using OpenAI Whisper API.
+ */
+declare class OpenAISTTAdapter implements STTAdapter {
+    private apiKey;
+    private model;
+    private baseURL;
+    private timeoutMs;
+    constructor(config: OpenAISTTConfig);
+    transcribe(audio: Buffer | ArrayBuffer, format?: string): Promise<string>;
+}
+
+interface OpenAITTSConfig {
+    apiKey: string;
+    model?: string;
+    voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
+    baseURL?: string;
+    timeoutMs?: number;
+}
+/**
+ * TTS adapter using OpenAI TTS API.
+ */
+declare class OpenAITTSAdapter implements TTSAdapter {
+    private apiKey;
+    private model;
+    private defaultVoice;
+    private baseURL;
+    private timeoutMs;
+    constructor(config: OpenAITTSConfig);
+    synthesize(text: string, options?: TTSSynthesizeOptions): Promise<Buffer>;
+    synthesizeStream(text: string, options?: TTSSynthesizeOptions): AsyncIterable<Buffer>;
+}
+
+export { OpenAISTTAdapter, type OpenAISTTConfig, OpenAITTSAdapter, type OpenAITTSConfig, RealtimeAgent, type RealtimeAgentConfig, type RealtimeAgentOptions, type RealtimeEvent, type RealtimeProcessResult, type STTAdapter, type TTSAdapter, type TTSSynthesizeOptions };

package/dist/index.js

@@ -0,0 +1,227 @@
+import { StreamingToolAgent } from '@orka-js/agent';
+
+// src/realtime-agent.ts
+var RealtimeAgent = class {
+  config;
+  stt;
+  tts;
+  innerAgent;
+  constructor(options) {
+    this.config = options.config;
+    this.stt = options.stt;
+    this.tts = options.tts;
+    this.innerAgent = new StreamingToolAgent(
+      {
+        goal: options.config.goal,
+        systemPrompt: options.config.systemPrompt,
+        tools: options.tools ?? []
+      },
+      options.llm
+    );
+  }
+  /**
+   * Process audio: transcribe → run LLM → synthesize.
+   * Returns final transcript, response text and audio buffer.
+   */
+  async process(audio, audioFormat = "audio/wav") {
+    const transcript = await this.stt.transcribe(audio, audioFormat);
+    let response = "";
+    for await (const event of this.innerAgent.runStream(transcript)) {
+      if (event.type === "done") {
+        response = event.content;
+      }
+    }
+    let audioOutput;
+    if (this.config.tts !== false && this.tts && response) {
+      audioOutput = await this.tts.synthesize(response);
+    }
+    return { transcript, response, audio: audioOutput };
+  }
+  /**
+   * Process audio as a stream of events — yields transcript, tokens, audio chunks, and done.
+   */
+  async *processStream(audio, audioFormat = "audio/wav") {
+    let transcript;
+    try {
+      transcript = await this.stt.transcribe(audio, audioFormat);
+    } catch (error) {
+      yield { type: "error", error, message: error.message };
+      return;
+    }
+    yield { type: "transcript", text: transcript };
+    let fullResponse = "";
+    for await (const event of this.innerAgent.runStream(transcript)) {
+      switch (event.type) {
+        case "token":
+          fullResponse += event.token;
+          yield { type: "token", content: event.token };
+          break;
+        case "tool_call":
+          yield { type: "tool_call", name: event.name, args: event.arguments };
+          break;
+        case "tool_result":
+          yield { type: "tool_result", name: String(event.toolCallId), result: event.result };
+          break;
+        case "done":
+          fullResponse = event.content || fullResponse;
+          break;
+        case "error":
+          yield { type: "error", error: event.error, message: event.error.message };
+          return;
+      }
+    }
+    let audioOutput;
+    if (this.config.tts !== false && this.tts && fullResponse) {
+      if (this.tts.synthesizeStream) {
+        for await (const chunk of this.tts.synthesizeStream(fullResponse)) {
+          yield { type: "audio_chunk", data: chunk };
+          if (!audioOutput) audioOutput = chunk;
+          else audioOutput = Buffer.concat([audioOutput, chunk]);
+        }
+      } else {
+        audioOutput = await this.tts.synthesize(fullResponse);
+        yield { type: "audio_chunk", data: audioOutput };
+      }
+    }
+    yield { type: "done", transcript, response: fullResponse, audio: audioOutput };
+  }
+  /**
+   * Returns an Express/Node.js-compatible WebSocket message handler.
+   * Expects binary audio messages from the client, sends back JSON events.
+   *
+   * @example
+   * ```typescript
+   * import { WebSocketServer } from 'ws';
+   * const wss = new WebSocketServer({ port: 8080 });
+   * wss.on('connection', agent.wsHandler());
+   * ```
+   */
+  wsHandler() {
+    return (ws) => {
+      ws.on("message", async (data) => {
+        const audio = data instanceof Buffer ? data : data instanceof ArrayBuffer ? Buffer.from(data) : Buffer.from(data);
+        try {
+          for await (const event of this.processStream(audio)) {
+            ws.send(JSON.stringify(event));
+          }
+        } catch (error) {
+          ws.send(JSON.stringify({
+            type: "error",
+            error: { message: error.message },
+            message: error.message
+          }));
+        }
+      });
+    };
+  }
+};
+
+// src/adapters/openai-stt.ts
+var OpenAISTTAdapter = class {
+  apiKey;
+  model;
+  baseURL;
+  timeoutMs;
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.model = config.model ?? "whisper-1";
+    this.baseURL = config.baseURL ?? "https://api.openai.com/v1";
+    this.timeoutMs = config.timeoutMs ?? 12e4;
+  }
+  async transcribe(audio, format = "audio/wav") {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+    const ext = format.includes("webm") ? "webm" : format.includes("mp3") ? "mp3" : format.includes("ogg") ? "ogg" : "wav";
+    const audioData = audio instanceof Buffer ? audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength) : audio;
+    const blob = new Blob([audioData], { type: format });
+    const formData = new FormData();
+    formData.append("file", blob, `audio.${ext}`);
+    formData.append("model", this.model);
+    formData.append("response_format", "text");
+    let response;
+    try {
+      response = await fetch(`${this.baseURL}/audio/transcriptions`, {
+        method: "POST",
+        signal: controller.signal,
+        headers: { "Authorization": `Bearer ${this.apiKey}` },
+        body: formData
+      });
+    } catch (error) {
+      clearTimeout(timeout);
+      if (error.name === "AbortError") {
+        throw new Error(`Whisper API request timed out after ${this.timeoutMs}ms`);
+      }
+      throw error;
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!response.ok) {
+      const err = await response.text();
+      throw new Error(`Whisper API error: ${response.status} - ${err}`);
+    }
+    return response.text();
+  }
+};
+
+// src/adapters/openai-tts.ts
+var OpenAITTSAdapter = class {
+  apiKey;
+  model;
+  defaultVoice;
+  baseURL;
+  timeoutMs;
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.model = config.model ?? "tts-1";
+    this.defaultVoice = config.voice ?? "alloy";
+    this.baseURL = config.baseURL ?? "https://api.openai.com/v1";
+    this.timeoutMs = config.timeoutMs ?? 6e4;
+  }
+  async synthesize(text, options = {}) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+    let response;
+    try {
+      response = await fetch(`${this.baseURL}/audio/speech`, {
+        method: "POST",
+        signal: controller.signal,
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({
+          model: this.model,
+          input: text,
+          voice: options.voice ?? this.defaultVoice,
+          response_format: options.format ?? "mp3",
+          speed: options.speed ?? 1
+        })
+      });
+    } catch (error) {
+      clearTimeout(timeout);
+      if (error.name === "AbortError") {
+        throw new Error(`TTS API request timed out after ${this.timeoutMs}ms`);
+      }
+      throw error;
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!response.ok) {
+      const err = await response.text();
+      throw new Error(`TTS API error: ${response.status} - ${err}`);
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    return Buffer.from(arrayBuffer);
+  }
+  async *synthesizeStream(text, options = {}) {
+    const sentences = text.match(/[^.!?]+[.!?]+/g) ?? [text];
+    for (const sentence of sentences) {
+      const audio = await this.synthesize(sentence.trim(), options);
+      yield audio;
+    }
+  }
+};
+
+export { OpenAISTTAdapter, OpenAITTSAdapter, RealtimeAgent };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/realtime-agent.ts","../src/adapters/openai-stt.ts","../src/adapters/openai-tts.ts"],"names":[],"mappings":";;;AAoCO,IAAM,gBAAN,MAAoB;AAAA,EACjB,MAAA;AAAA,EACA,GAAA;AAAA,EACA,GAAA;AAAA,EACA,UAAA;AAAA,EAER,YAAY,OAAA,EAA+B;AACzC,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,MAAM,OAAA,CAAQ,GAAA;AACnB,IAAA,IAAA,CAAK,MAAM,OAAA,CAAQ,GAAA;AAEnB,IAAA,IAAA,CAAK,aAAa,IAAI,kBAAA;AAAA,MACpB;AAAA,QACE,IAAA,EAAM,QAAQ,MAAA,CAAO,IAAA;AAAA,QACrB,YAAA,EAAc,QAAQ,MAAA,CAAO,YAAA;AAAA,QAC7B,KAAA,EAAO,OAAA,CAAQ,KAAA,IAAS;AAAC,OAC3B;AAAA,MACA,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAA,CAAQ,KAAA,EAA6B,WAAA,GAAc,WAAA,EAA6C;AACpG,IAAA,MAAM,aAAa,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,OAAO,WAAW,CAAA;AAE/D,IAAA,IAAI,QAAA,GAAW,EAAA;AACf,IAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,UAAA,CAAW,SAAA,CAAU,UAAU,CAAA,EAAG;AAC/D,MAAA,IAAI,KAAA,CAAM,SAAS,MAAA,EAAQ;AACzB,QAAA,QAAA,GAAW,KAAA,CAAM,OAAA;AAAA,MACnB;AAAA,IACF;AAEA,IAAA,IAAI,WAAA;AACJ,IAAA,IAAI,KAAK,MAAA,CAAO,GAAA,KAAQ,KAAA,IAAS,IAAA,CAAK,OAAO,QAAA,EAAU;AACrD,MAAA,WAAA,GAAc,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,QAAQ,CAAA;AAAA,IAClD;AAEA,IAAA,OAAO,EAAE,UAAA,EAAY,QAAA,EAAU,KAAA,EAAO,WAAA,EAAY;AAAA,EACpD;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,aAAA,CACL,KAAA,EACA,WAAA,GAAc,WAAA,EACgB;AAC9B,IAAA,IAAI,UAAA;AACJ,IAAA,IAAI;AACF,MAAA,UAAA,GAAa,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,OAAO,WAAW,CAAA;AAAA,IAC3D,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,EAAE,IAAA,EAAM,OAAA,EAAS,KAAA,EAAuB,OAAA,EAAU,MAAgB,OAAA,EAAQ;AAChF,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,IAAA,EAAM,YAAA,EAAc,IAAA,EAAM,UAAA,EAAW;AAE7C,IAAA,IAAI,YAAA,GAAe,EAAA;AAEnB,IAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,UAAA,CAAW,SAAA,CAAU,UAAU,CAAA,EAAG;AAC/D,MAAA,QAAQ,MAAM,IAAA;AAAM,QAClB,KAAK,OAAA;AACH,UAAA,YAAA,IAAgB,KAAA,CAAM,KAAA;AACtB,UAAA,MAAM,EAAE,IAAA,EAAM,OAAA,EAAS,OAAA,EAAS,MAAM,KAAA,EAAM;AAC5C,UAAA;AAAA,QACF,KAAK,WAAA;AACH,UAAA,MAAM,EAAE,MAAM,WAAA,EAAa,IAAA,EAAM,MAAM,IAAA,EAAM,IAAA,EAAM,MAAM,SAAA,EAAU;AACnE,UAAA;AAAA,QACF,KAAK,aAAA;AACH,UAAA,MAAM,EAAE,IAAA,EAAM,aAAA,EAAe,IAAA,EAAM,MAAA,CAAO,MAAM,UAAU,CAAA,EAAG,MAAA,EAAQ,KAAA,CAAM,MAAA,EAAO;AAClF,UAAA;AAAA,QACF,KAAK,MAAA;AACH,UAAA,YAAA,GAAe,MAAM,OAAA,IAAW,YAAA;AAChC,UAAA;AAAA,QACF,KAAK,OAAA;AACH,UAAA,MAAM,EAAE,MAAM,OAAA,EAAS,KAAA,EAAO,MAAM,KAAA,EAAO,OAAA,EAAS,KAAA,CAAM,KAAA,CAAM,OAAA,EAAQ;AACxE,UAAA;AAAA;AACJ,IACF;AAGA,IAAA,IAAI,WAAA;AACJ,IAAA,IAAI,KAAK,MAAA,CAAO,GAAA,KAAQ,KAAA,IAAS,IAAA,CAAK,OAAO,YAAA,EAAc;AACzD,MAAA,IAAI,IAAA,CAAK,IAAI,gBAAA,EAAkB;AAC7B,QAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,GAAA,CAAI,gBAAA,CAAiB,YAAY,CAAA,EAAG;AACjE,UAAA,MAAM,EAAE,IAAA,EAAM,aAAA,EAAe,IAAA,EAAM,KAAA,EAAM;AACzC,UAAA,IAAI,CAAC,aAAa,WAAA,GAAc,KAAA;AAAA,6BACb,MAAA,CAAO,MAAA,CAAO,CAAC,WAAA,EAAa,KAAK,CAAC,CAAA;AAAA,QACvD;AAAA,MACF,CAAA,MAAO;AACL,QAAA,WAAA,GAAc,MAAM,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,YAAY,CAAA;AACpD,QAAA,MAAM,EAAE,IAAA,EAAM,aAAA,EAAe,IAAA,EAAM,WAAA,EAAY;AAAA,MACjD;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,IAAA,EAAM,MAAA,EAAQ,YAAY,QAAA,EAAU,YAAA,EAAc,OAAO,WAAA,EAAY;AAAA,EAC/E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,SAAA,GAAY;AACV,IAAA,OAAO,CAAC,EAAA,KAGF;AACJ,MAAA,EAAA,CAAG,EAAA,CAAG,SAAA,EAAW,OAAO,IAAA,KAAkB;AACxC,QAAA,MAAM,KAAA,GAAQ,IAAA,YAAgB,MAAA,GAAS,IAAA,GACnC,IAAA,YAAgB,WAAA,GAAc,MAAA,CAAO,IAAA,CAAK,IAAI,CAAA,GAC9C,MAAA,CAAO,IAAA,CAAK,IAAkB,CAAA;AAElC,QAAA,IAAI;AACF,UAAA,WAAA,MAAiB,KAAA,IAAS,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA,EAAG;AACnD,YAAA,EAAA,CAAG,IAAA,CAAK,IAAA,CAAK,SAAA,CAAU,KAAK,CAAC,CAAA;AAAA,UAC/B;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,EAAA,CAAG,IAAA,CAAK,KAAK,SAAA,CAAU;AAAA,YACrB,IAAA,EAAM,OAAA;AAAA,YACN,KAAA,EAAO,EAAE,OAAA,EAAU,KAAA,CAAgB,OAAA,EAAQ;AAAA,YAC3C,SAAU,KAAA,CAAgB;AAAA,WAC3B,CAAC,CAAA;AAAA,QACJ;AAAA,MACF,CAAC,CAAA;AAAA,IACH,CAAA;AAAA,EACF;AACF;;;AChKO,IAAM,mBAAN,MAA6C;AAAA,EAC1C,MAAA;AAAA,EACA,KAAA;AAAA,EACA,OAAA;AAAA,EACA,SAAA;AAAA,EAER,YAAY,MAAA,EAAyB;AACnC,IAAA,IAAA,CAAK,SAAS,MAAA,CAAO,MAAA;AACrB,IAAA,IAAA,CAAK,KAAA,GAAQ,OAAO,KAAA,IAAS,WAAA;AAC7B,IAAA,IAAA,CAAK,OAAA,GAAU,OAAO,OAAA,IAAW,2BAAA;AACjC,IAAA,IAAA,CAAK,SAAA,GAAY,OAAO,SAAA,IAAa,IAAA;AAAA,EACvC;AAAA,EAEA,MAAM,UAAA,CAAW,KAAA,EAA6B,MAAA,GAAS,WAAA,EAA8B;AACnF,IAAA,MAAM,UAAA,GAAa,IAAI,eAAA,EAAgB;AACvC,IAAA,MAAM,UAAU,UAAA,CAAW,MAAM,WAAW,KAAA,EAAM,EAAG,KAAK,SAAS,CAAA;AAEnE,IAAA,MAAM,GAAA,GAAM,MAAA,CAAO,QAAA,CAAS,MAAM,IAAI,MAAA,GAClC,MAAA,CAAO,QAAA,CAAS,KAAK,IAAI,KAAA,GACzB,MAAA,CAAO,QAAA,CAAS,KAAK,IAAI,KAAA,GACzB,KAAA;AAEJ,IAAA,MAAM,SAAA,GAAY,KAAA,YAAiB,MAAA,GAAS,KAAA,CAAM,MAAA,CAAO,KAAA,CAAM,KAAA,CAAM,UAAA,EAAY,KAAA,CAAM,UAAA,GAAa,KAAA,CAAM,UAAU,CAAA,GAAI,KAAA;AACxH,IAAA,MAAM,IAAA,GAAO,IAAI,IAAA,CAAK,CAAC,SAAqB,CAAA,EAAG,EAAE,IAAA,EAAM,MAAA,EAAQ,CAAA;AAC/D,IAAA,MAAM,QAAA,GAAW,IAAI,QAAA,EAAS;AAC9B,IAAA,QAAA,CAAS,MAAA,CAAO,MAAA,EAAQ,IAAA,EAAM,CAAA,MAAA,EAAS,GAAG,CAAA,CAAE,CAAA;AAC5C,IAAA,QAAA,CAAS,MAAA,CAAO,OAAA,EAAS,IAAA,CAAK,KAAK,CAAA;AACnC,IAAA,QAAA,CAAS,MAAA,CAAO,mBAAmB,MAAM,CAAA;AAEzC,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,OAAO,CAAA,qBAAA,CAAA,EAAyB;AAAA,QAC7D,MAAA,EAAQ,MAAA;AAAA,QACR,QAAQ,UAAA,CAAW,MAAA;AAAA,QACnB,SAAS,EAAE,eAAA,EAAiB,CAAA,OAAA,EAAU,IAAA,CAAK,MAAM,CAAA,CAAA,EAAG;AAAA,QACpD,IAAA,EAAM;AAAA,OACP,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,IAAK,KAAA,CAAgB,SAAS,YAAA,EAAc;AAC1C,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,oCAAA,EAAuC,IAAA,CAAK,SAAS,CAAA,EAAA,CAAI,CAAA;AAAA,MAC3E;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,YAAA,CAAa,OAAO,CAAA;AAAA,IACtB;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,GAAA,GAAM,MAAM,QAAA,CAAS,IAAA,EAAK;AAChC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mBAAA,EAAsB,SAAS,MAAM,CAAA,GAAA,EAAM,GAAG,CAAA,CAAE,CAAA;AAAA,IAClE;AAEA,IAAA,OAAO,SAAS,IAAA,EAAK;AAAA,EACvB;AACF;;;ACrDO,IAAM,mBAAN,MAA6C;AAAA,EAC1C,MAAA;AAAA,EACA,KAAA;AAAA,EACA,YAAA;AAAA,EACA,OAAA;AAAA,EACA,SAAA;AAAA,EAER,YAAY,MAAA,EAAyB;AACnC,IAAA,IAAA,CAAK,SAAS,MAAA,CAAO,MAAA;AACrB,IAAA,IAAA,CAAK,KAAA,GAAQ,OAAO,KAAA,IAAS,OAAA;AAC7B,IAAA,IAAA,CAAK,YAAA,GAAe,OAAO,KAAA,IAAS,OAAA;AACpC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAO,OAAA,IAAW,2BAAA;AACjC,IAAA,IAAA,CAAK,SAAA,GAAY,OAAO,SAAA,IAAa,GAAA;AAAA,EACvC;AAAA,EAEA,MAAM,UAAA,CAAW,IAAA,EAAc,OAAA,GAAgC,EAAC,EAAoB;AAClF,IAAA,MAAM,UAAA,GAAa,IAAI,eAAA,EAAgB;AACvC,IAAA,MAAM,UAAU,UAAA,CAAW,MAAM,WAAW,KAAA,EAAM,EAAG,KAAK,SAAS,CAAA;AAEnE,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,OAAO,CAAA,aAAA,CAAA,EAAiB;AAAA,QACrD,MAAA,EAAQ,MAAA;AAAA,QACR,QAAQ,UAAA,CAAW,MAAA;AAAA,QACnB,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB,kBAAA;AAAA,UAChB,eAAA,EAAiB,CAAA,OAAA,EAAU,IAAA,CAAK,MAAM,CAAA;AAAA,SACxC;AAAA,QACA,IAAA,EAAM,KAAK,SAAA,CAAU;AAAA,UACnB,OAAO,IAAA,CAAK,KAAA;AAAA,UACZ,KAAA,EAAO,IAAA;AAAA,UACP,KAAA,EAAO,OAAA,CAAQ,KAAA,IAAS,IAAA,CAAK,YAAA;AAAA,UAC7B,eAAA,EAAiB,QAAQ,MAAA,IAAU,KAAA;AAAA,UACnC,KAAA,EAAO,QAAQ,KAAA,IAAS;AAAA,SACzB;AAAA,OACF,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,OAAO,CAAA;AACpB,MAAA,IAAK,KAAA,CAAgB,SAAS,YAAA,EAAc;AAC1C,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,gCAAA,EAAmC,IAAA,CAAK,SAAS,CAAA,EAAA,CAAI,CAAA;AAAA,MACvE;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,YAAA,CAAa,OAAO,CAAA;AAAA,IACtB;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAM,GAAA,GAAM,MAAM,QAAA,CAAS,IAAA,EAAK;AAChC,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,eAAA,EAAkB,SAAS,MAAM,CAAA,GAAA,EAAM,GAAG,CAAA,CAAE,CAAA;AAAA,IAC9D;AAEA,IAAA,MAAM,WAAA,GAAc,MAAM,QAAA,CAAS,WAAA,EAAY;AAC/C,IAAA,OAAO,MAAA,CAAO,KAAK,WAAW,CAAA;AAAA,EAChC;AAAA,EAEA,OAAO,gBAAA,CAAiB,IAAA,EAAc,OAAA,GAAgC,EAAC,EAA0B;AAE/F,IAAA,MAAM,YAAY,IAAA,CAAK,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,IAAI,CAAA;AACvD,IAAA,KAAA,MAAW,YAAY,SAAA,EAAW;AAChC,MAAA,MAAM,QAAQ,MAAM,IAAA,CAAK,WAAW,QAAA,CAAS,IAAA,IAAQ,OAAO,CAAA;AAC5D,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF","file":"index.js","sourcesContent":["import type { LLMAdapter } from '@orka-js/core';\nimport { StreamingToolAgent } from '@orka-js/agent';\nimport type { Tool } from '@orka-js/agent';\nimport type {\n  STTAdapter,\n  TTSAdapter,\n  RealtimeAgentConfig,\n  RealtimeEvent,\n  RealtimeProcessResult,\n} from './types.js';\n\nexport interface RealtimeAgentOptions {\n  config: RealtimeAgentConfig;\n  llm: LLMAdapter;\n  stt: STTAdapter;\n  tts?: TTSAdapter;\n  tools?: Tool[];\n}\n\n/**\n * A voice agent that processes audio through the STT → LLM → TTS pipeline.\n *\n * @example\n * ```typescript\n * const agent = new RealtimeAgent({\n *   config: { goal: 'Answer questions', tts: true },\n *   llm: new OpenAIAdapter({ apiKey }),\n *   stt: new OpenAISTTAdapter({ apiKey }),\n *   tts: new OpenAITTSAdapter({ apiKey }),\n * });\n *\n * const result = await agent.process(audioBuffer);\n * console.log(result.transcript); // what the user said\n * console.log(result.response);   // the LLM's reply\n * ```\n */\nexport class RealtimeAgent {\n  private config: RealtimeAgentConfig;\n  private stt: STTAdapter;\n  private tts?: TTSAdapter;\n  private innerAgent: StreamingToolAgent;\n\n  constructor(options: RealtimeAgentOptions) {\n    this.config = options.config;\n    this.stt = options.stt;\n    this.tts = options.tts;\n\n    this.innerAgent = new StreamingToolAgent(\n      {\n        goal: options.config.goal,\n        systemPrompt: options.config.systemPrompt,\n        tools: options.tools ?? [],\n      },\n      options.llm,\n    );\n  }\n\n  /**\n   * Process audio: transcribe → run LLM → synthesize.\n   * Returns final transcript, response text and audio buffer.\n   */\n  async process(audio: Buffer | ArrayBuffer, audioFormat = 'audio/wav'): Promise<RealtimeProcessResult> {\n    const transcript = await this.stt.transcribe(audio, audioFormat);\n\n    let response = '';\n    for await (const event of this.innerAgent.runStream(transcript)) {\n      if (event.type === 'done') {\n        response = event.content;\n      }\n    }\n\n    let audioOutput: Buffer | undefined;\n    if (this.config.tts !== false && this.tts && response) {\n      audioOutput = await this.tts.synthesize(response);\n    }\n\n    return { transcript, response, audio: audioOutput };\n  }\n\n  /**\n   * Process audio as a stream of events — yields transcript, tokens, audio chunks, and done.\n   */\n  async *processStream(\n    audio: Buffer | ArrayBuffer,\n    audioFormat = 'audio/wav',\n  ): AsyncIterable<RealtimeEvent> {\n    let transcript: string;\n    try {\n      transcript = await this.stt.transcribe(audio, audioFormat);\n    } catch (error) {\n      yield { type: 'error', error: error as Error, message: (error as Error).message };\n      return;\n    }\n\n    yield { type: 'transcript', text: transcript };\n\n    let fullResponse = '';\n\n    for await (const event of this.innerAgent.runStream(transcript)) {\n      switch (event.type) {\n        case 'token':\n          fullResponse += event.token;\n          yield { type: 'token', content: event.token };\n          break;\n        case 'tool_call':\n          yield { type: 'tool_call', name: event.name, args: event.arguments };\n          break;\n        case 'tool_result':\n          yield { type: 'tool_result', name: String(event.toolCallId), result: event.result };\n          break;\n        case 'done':\n          fullResponse = event.content || fullResponse;\n          break;\n        case 'error':\n          yield { type: 'error', error: event.error, message: event.error.message };\n          return;\n      }\n    }\n\n    // Synthesize audio\n    let audioOutput: Buffer | undefined;\n    if (this.config.tts !== false && this.tts && fullResponse) {\n      if (this.tts.synthesizeStream) {\n        for await (const chunk of this.tts.synthesizeStream(fullResponse)) {\n          yield { type: 'audio_chunk', data: chunk };\n          if (!audioOutput) audioOutput = chunk;\n          else audioOutput = Buffer.concat([audioOutput, chunk]);\n        }\n      } else {\n        audioOutput = await this.tts.synthesize(fullResponse);\n        yield { type: 'audio_chunk', data: audioOutput };\n      }\n    }\n\n    yield { type: 'done', transcript, response: fullResponse, audio: audioOutput };\n  }\n\n  /**\n   * Returns an Express/Node.js-compatible WebSocket message handler.\n   * Expects binary audio messages from the client, sends back JSON events.\n   *\n   * @example\n   * ```typescript\n   * import { WebSocketServer } from 'ws';\n   * const wss = new WebSocketServer({ port: 8080 });\n   * wss.on('connection', agent.wsHandler());\n   * ```\n   */\n  wsHandler() {\n    return (ws: {\n      on(event: string, listener: (...args: unknown[]) => void): void;\n      send(data: string): void;\n    }) => {\n      ws.on('message', async (data: unknown) => {\n        const audio = data instanceof Buffer ? data\n          : data instanceof ArrayBuffer ? Buffer.from(data)\n          : Buffer.from(data as Uint8Array);\n\n        try {\n          for await (const event of this.processStream(audio)) {\n            ws.send(JSON.stringify(event));\n          }\n        } catch (error) {\n          ws.send(JSON.stringify({\n            type: 'error',\n            error: { message: (error as Error).message },\n            message: (error as Error).message,\n          }));\n        }\n      });\n    };\n  }\n}\n","import type { STTAdapter } from '../types.js';\n\nexport interface OpenAISTTConfig {\n  apiKey: string;\n  model?: string;\n  baseURL?: string;\n  timeoutMs?: number;\n}\n\n/**\n * STT adapter using OpenAI Whisper API.\n */\nexport class OpenAISTTAdapter implements STTAdapter {\n  private apiKey: string;\n  private model: string;\n  private baseURL: string;\n  private timeoutMs: number;\n\n  constructor(config: OpenAISTTConfig) {\n    this.apiKey = config.apiKey;\n    this.model = config.model ?? 'whisper-1';\n    this.baseURL = config.baseURL ?? 'https://api.openai.com/v1';\n    this.timeoutMs = config.timeoutMs ?? 120_000;\n  }\n\n  async transcribe(audio: Buffer | ArrayBuffer, format = 'audio/wav'): Promise<string> {\n    const controller = new AbortController();\n    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);\n\n    const ext = format.includes('webm') ? 'webm'\n      : format.includes('mp3') ? 'mp3'\n      : format.includes('ogg') ? 'ogg'\n      : 'wav';\n\n    const audioData = audio instanceof Buffer ? audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength) : audio;\n    const blob = new Blob([audioData as BlobPart], { type: format });\n    const formData = new FormData();\n    formData.append('file', blob, `audio.${ext}`);\n    formData.append('model', this.model);\n    formData.append('response_format', 'text');\n\n    let response: Response;\n    try {\n      response = await fetch(`${this.baseURL}/audio/transcriptions`, {\n        method: 'POST',\n        signal: controller.signal,\n        headers: { 'Authorization': `Bearer ${this.apiKey}` },\n        body: formData,\n      });\n    } catch (error) {\n      clearTimeout(timeout);\n      if ((error as Error).name === 'AbortError') {\n        throw new Error(`Whisper API request timed out after ${this.timeoutMs}ms`);\n      }\n      throw error;\n    } finally {\n      clearTimeout(timeout);\n    }\n\n    if (!response.ok) {\n      const err = await response.text();\n      throw new Error(`Whisper API error: ${response.status} - ${err}`);\n    }\n\n    return response.text();\n  }\n}\n","import type { TTSAdapter, TTSSynthesizeOptions } from '../types.js';\n\nexport interface OpenAITTSConfig {\n  apiKey: string;\n  model?: string;\n  voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';\n  baseURL?: string;\n  timeoutMs?: number;\n}\n\n/**\n * TTS adapter using OpenAI TTS API.\n */\nexport class OpenAITTSAdapter implements TTSAdapter {\n  private apiKey: string;\n  private model: string;\n  private defaultVoice: string;\n  private baseURL: string;\n  private timeoutMs: number;\n\n  constructor(config: OpenAITTSConfig) {\n    this.apiKey = config.apiKey;\n    this.model = config.model ?? 'tts-1';\n    this.defaultVoice = config.voice ?? 'alloy';\n    this.baseURL = config.baseURL ?? 'https://api.openai.com/v1';\n    this.timeoutMs = config.timeoutMs ?? 60_000;\n  }\n\n  async synthesize(text: string, options: TTSSynthesizeOptions = {}): Promise<Buffer> {\n    const controller = new AbortController();\n    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);\n\n    let response: Response;\n    try {\n      response = await fetch(`${this.baseURL}/audio/speech`, {\n        method: 'POST',\n        signal: controller.signal,\n        headers: {\n          'Content-Type': 'application/json',\n          'Authorization': `Bearer ${this.apiKey}`,\n        },\n        body: JSON.stringify({\n          model: this.model,\n          input: text,\n          voice: options.voice ?? this.defaultVoice,\n          response_format: options.format ?? 'mp3',\n          speed: options.speed ?? 1.0,\n        }),\n      });\n    } catch (error) {\n      clearTimeout(timeout);\n      if ((error as Error).name === 'AbortError') {\n        throw new Error(`TTS API request timed out after ${this.timeoutMs}ms`);\n      }\n      throw error;\n    } finally {\n      clearTimeout(timeout);\n    }\n\n    if (!response.ok) {\n      const err = await response.text();\n      throw new Error(`TTS API error: ${response.status} - ${err}`);\n    }\n\n    const arrayBuffer = await response.arrayBuffer();\n    return Buffer.from(arrayBuffer);\n  }\n\n  async *synthesizeStream(text: string, options: TTSSynthesizeOptions = {}): AsyncIterable<Buffer> {\n    // Split text into sentences for lower latency\n    const sentences = text.match(/[^.!?]+[.!?]+/g) ?? [text];\n    for (const sentence of sentences) {\n      const audio = await this.synthesize(sentence.trim(), options);\n      yield audio;\n    }\n  }\n}\n"]}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@orka-js/realtime",
+  "version": "1.5.0",
+  "description": "Voice agent for OrkaJS — STT → LLM → TTS pipeline with WebSocket support",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@orka-js/core": "1.5.0",
+    "@orka-js/agent": "1.5.2"
+  },
+  "peerDependencies": {
+    "ws": ">=8.0.0"
+  },
+  "peerDependenciesMeta": {
+    "ws": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/ws": "^8.5.10",
+    "vitest": "^1.6.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  }
+}