@livekit/agents 0.5.1 → 0.6.0

package/dist/stt/stream_adapter.d.ts

@@ -4,12 +4,15 @@ import type { SpeechEvent } from './stt.js';
 import { STT, SpeechStream } from './stt.js';
 export declare class StreamAdapter extends STT {
     #private;
+    label: string;
     constructor(stt: STT, vad: VAD);
-    recognize(frame: AudioFrame): Promise<SpeechEvent>;
+    _recognize(frame: AudioFrame): Promise<SpeechEvent>;
     stream(): StreamAdapterWrapper;
 }
 export declare class StreamAdapterWrapper extends SpeechStream {
     #private;
+    label: string;
     constructor(stt: STT, vad: VAD);
+    monitorMetrics(): Promise<void>;
 }
 //# sourceMappingURL=stream_adapter.d.ts.map

package/dist/stt/stream_adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/stt/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,GAAG,EAAa,MAAM,WAAW,CAAC;AAEhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAmB,YAAY,EAAE,MAAM,UAAU,CAAC;AAE9D,qBAAa,aAAc,SAAQ,GAAG;;gBAIxB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IAM9B,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAIlD,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,YAAY;;gBAIxC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;CA0C/B"}
+{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/stt/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,GAAG,EAAa,MAAM,WAAW,CAAC;AAEhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAmB,YAAY,EAAE,MAAM,UAAU,CAAC;AAE9D,qBAAa,aAAc,SAAQ,GAAG;;IAGpC,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IAW9B,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAInD,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,YAAY;;IAGpD,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IASxB,cAAc;CAsCrB"}

package/dist/stt/stream_adapter.js

@@ -3,12 +3,17 @@ import { STT, SpeechEventType, SpeechStream } from "./stt.js";
 class StreamAdapter extends STT {
   #stt;
   #vad;
+  label;
   constructor(stt, vad) {
     super({ streaming: true, interimResults: false });
     this.#stt = stt;
     this.#vad = vad;
+    this.label = `stt.StreamAdapter<${this.#stt.label}>`;
+    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);
+    });
   }
-  recognize(frame) {
+  _recognize(frame) {
     return this.#stt.recognize(frame);
   }
   stream() {
@@ -18,12 +23,17 @@ class StreamAdapter extends STT {
 class StreamAdapterWrapper extends SpeechStream {
   #stt;
   #vadStream;
+  label;
   constructor(stt, vad) {
-    super();
+    super(stt);
     this.#stt = stt;
     this.#vadStream = vad.stream();
+    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;
     this.#run();
   }
+  async monitorMetrics() {
+    return;
+  }
   async #run() {
     const forwardInput = async () => {
       for await (const input of this.input) {
@@ -39,15 +49,15 @@ class StreamAdapterWrapper extends SpeechStream {
       for await (const ev of this.#vadStream) {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
-            this.queue.put({ type: SpeechEventType.START_OF_SPEECH });
+            this.output.put({ type: SpeechEventType.START_OF_SPEECH });
             break;
           case VADEventType.END_OF_SPEECH:
-            this.queue.put({ type: SpeechEventType.END_OF_SPEECH });
+            this.output.put({ type: SpeechEventType.END_OF_SPEECH });
             const event = await this.#stt.recognize(ev.frames);
             if (!event.alternatives[0].text) {
               continue;
             }
-            this.queue.put(event);
+            this.output.put(event);
             break;
         }
       }

package/dist/stt/stream_adapter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n  }\n\n  recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n\n  constructor(stt: STT, vad: VAD) {\n    super();\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n\n    this.#run();\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.queue.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.queue.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.queue.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,KAAK,iBAAiB,oBAAoB;AAE5C,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,UAAU,OAAyC;AACjD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,aAAa;AAAA,EACrD;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAE7B,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,aAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,MAAM,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AACxD;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,MAAM,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAEtD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,MAAM,IAAI,KAAK;AACpB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}
+{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n    this.label = `stt.StreamAdapter<${this.#stt.label}>`;\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    });\n  }\n\n  _recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super(stt);\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;\n\n    this.#run();\n  }\n\n  async monitorMetrics() {\n    return; // do nothing\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.output.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,KAAK,iBAAiB,oBAAoB;AAE5C,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,gBAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,gBAAgB,mBAAmB,OAAO;AAAA,IACtD,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,OAAyC;AAClD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,aAAa;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAC7B,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAExD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,iBAAiB;AACrB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,aAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AACzD;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAEvD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,OAAO,IAAI,KAAK;AACrB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}

package/dist/stt/stt.cjs

@@ -23,29 +23,72 @@ __export(stt_exports, {
   SpeechStream: () => SpeechStream
 });
 module.exports = __toCommonJS(stt_exports);
+var import_node_events = require("node:events");
 var import_utils = require("../utils.cjs");
 var SpeechEventType = /* @__PURE__ */ ((SpeechEventType2) => {
   SpeechEventType2[SpeechEventType2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
   SpeechEventType2[SpeechEventType2["INTERIM_TRANSCRIPT"] = 1] = "INTERIM_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["FINAL_TRANSCRIPT"] = 2] = "FINAL_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["END_OF_SPEECH"] = 3] = "END_OF_SPEECH";
+  SpeechEventType2[SpeechEventType2["RECOGNITION_USAGE"] = 4] = "RECOGNITION_USAGE";
+  SpeechEventType2[SpeechEventType2["METRICS_COLLECTED"] = 5] = "METRICS_COLLECTED";
   return SpeechEventType2;
 })(SpeechEventType || {});
-class STT {
+class STT extends import_node_events.EventEmitter {
   #capabilities;
   constructor(capabilities) {
+    super();
     this.#capabilities = capabilities;
   }
   /** Returns this STT's capabilities */
   get capabilities() {
     return this.#capabilities;
   }
+  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
+  async recognize(frame) {
+    const startTime = process.hrtime.bigint();
+    const event = await this._recognize(frame);
+    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1e6));
+    this.emit(5 /* METRICS_COLLECTED */, {
+      requestId: event.requestId ?? "",
+      timestamp: Date.now(),
+      duration,
+      label: this.label,
+      audioDuration: Array.isArray(frame) ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0) : frame.samplesPerChannel / frame.sampleRate,
+      streamed: false
+    });
+    return event;
+  }
 }
 class SpeechStream {
   static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
   input = new import_utils.AsyncIterableQueue();
+  output = new import_utils.AsyncIterableQueue();
   queue = new import_utils.AsyncIterableQueue();
   closed = false;
+  #stt;
+  constructor(stt) {
+    this.#stt = stt;
+    this.monitorMetrics();
+  }
+  async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    for await (const event of this.queue) {
+      this.output.put(event);
+      if (event.type !== 4 /* RECOGNITION_USAGE */) continue;
+      const duration = process.hrtime.bigint() - startTime;
+      const metrics = {
+        timestamp: Date.now(),
+        requestId: event.requestId,
+        duration: Math.trunc(Number(duration / BigInt(1e6))),
+        label: this.label,
+        audioDuration: event.recognitionUsage.audioDuration,
+        streamed: true
+      };
+      this.#stt.emit(5 /* METRICS_COLLECTED */, metrics);
+    }
+    this.output.close();
+  }
   /** Push an audio frame to the STT */
   pushFrame(frame) {
     if (this.input.closed) {
@@ -77,12 +120,13 @@ class SpeechStream {
     this.input.close();
   }
   next() {
-    return this.queue.next();
+    return this.output.next();
   }
   /** Close both the input and output of the STT stream */
   close() {
     this.input.close();
     this.queue.close();
+    this.output.close();
     this.closed = true;
   }
   [Symbol.asyncIterator]() {

package/dist/stt/stt.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT {\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  abstract recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected closed = false;\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,mBAAmC;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AApBU,SAAAA;AAAA,GAAA;AAwDL,MAAe,IAAI;AAAA,EACxB;AAAA,EAEA,YAAY,cAA+B;AACzC,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAUF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAoE;AAAA,EAChF,QAAQ,IAAI,gCAAgC;AAAA,EAC5C,SAAS;AAAA;AAAA,EAGnB,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}
+{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { STTMetrics } from '../metrics/base.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  METRICS_COLLECTED = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport type STTCallbacks = {\n  [SpeechEventType.METRICS_COLLECTED]: (metrics: STTMetrics) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit(SpeechEventType.METRICS_COLLECTED, {\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      duration,\n      label: this.label,\n      audioDuration: Array.isArray(frame)\n        ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0)\n        : frame.samplesPerChannel / frame.sampleRate,\n      streamed: false,\n    });\n    return event;\n  }\n\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n\n  constructor(stt: STT) {\n    this.#stt = stt;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const duration = process.hrtime.bigint() - startTime;\n      const metrics: STTMetrics = {\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        duration: Math.trunc(Number(duration / BigInt(1000000))),\n        label: this.label,\n        audioDuration: event.recognitionUsage!.audioDuration,\n        streamed: true,\n      };\n      this.#stt.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    }\n    this.output.close();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAG7B,mBAAmC;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AACA,EAAAA,kCAAA,uBAAoB,KAApB;AAvBU,SAAAA;AAAA,GAAA;AAqEL,MAAe,YAAa,gCAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,WAAW,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AAC/E,SAAK,KAAK,2BAAmC;AAAA,MAC3C,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,eAAe,MAAM,QAAQ,KAAK,IAC9B,MAAM,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,oBAAoB,EAAE,YAAY,CAAC,IACpE,MAAM,oBAAoB,MAAM;AAAA,MACpC,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AASF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAoE;AAAA,EAChF,SAAS,IAAI,gCAAgC;AAAA,EAC7C,QAAQ,IAAI,gCAAgC;AAAA,EAE5C,SAAS;AAAA,EACnB;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AAExC,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,YAAM,UAAsB;AAAA,QAC1B,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,QACvD,OAAO,KAAK;AAAA,QACZ,eAAe,MAAM,iBAAkB;AAAA,QACvC,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,2BAAmC,OAAO;AAAA,IAC3D;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}

package/dist/stt/stt.d.ts

@@ -1,4 +1,6 @@
 import type { AudioFrame } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { STTMetrics } from '../metrics/base.js';
 import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue } from '../utils.js';
 /** Indicates start/middle/end of speech */
@@ -22,7 +24,10 @@ export declare enum SpeechEventType {
      * Indicate the end of speech, emitted when the user stops speaking.
      * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.
      */
-    END_OF_SPEECH = 3
+    END_OF_SPEECH = 3,
+    /** Usage event, emitted periodically to indicate usage metrics. */
+    RECOGNITION_USAGE = 4,
+    METRICS_COLLECTED = 5
 }
 /** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
@@ -32,10 +37,15 @@ export interface SpeechData {
     endTime: number;
     confidence: number;
 }
+export interface RecognitionUsage {
+    audioDuration: number;
+}
 /** SpeechEvent is a packet of speech-to-text data. */
 export interface SpeechEvent {
     type: SpeechEventType;
     alternatives?: [SpeechData, ...SpeechData[]];
+    requestId?: string;
+    recognitionUsage?: RecognitionUsage;
 }
 /**
  * Describes the capabilities of the STT provider.
@@ -47,6 +57,10 @@ export interface STTCapabilities {
     streaming: boolean;
     interimResults: boolean;
 }
+export type STTCallbacks = {
+    [SpeechEventType.METRICS_COLLECTED]: (metrics: STTMetrics) => void;
+};
+declare const STT_base: new () => TypedEmitter<STTCallbacks>;
 /**
  * An instance of a speech-to-text adapter.
  *
@@ -54,13 +68,15 @@ export interface STTCapabilities {
  * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
  * exports its own child STT class, which inherits this class's methods.
  */
-export declare abstract class STT {
+export declare abstract class STT extends STT_base {
     #private;
+    abstract label: string;
     constructor(capabilities: STTCapabilities);
     /** Returns this STT's capabilities */
     get capabilities(): STTCapabilities;
     /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
-    abstract recognize(frame: AudioBuffer): Promise<SpeechEvent>;
+    recognize(frame: AudioBuffer): Promise<SpeechEvent>;
+    protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;
     /**
      * Returns a {@link SpeechStream} that can be used to push audio frames and receive
      * transcriptions
@@ -84,10 +100,15 @@ export declare abstract class STT {
  * exports its own child SpeechStream class, which inherits this class's methods.
  */
 export declare abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {
+    #private;
     protected static readonly FLUSH_SENTINEL: unique symbol;
     protected input: AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>;
+    protected output: AsyncIterableQueue<SpeechEvent>;
     protected queue: AsyncIterableQueue<SpeechEvent>;
+    abstract label: string;
     protected closed: boolean;
+    constructor(stt: STT);
+    protected monitorMetrics(): Promise<void>;
     /** Push an audio frame to the STT */
     pushFrame(frame: AudioFrame): void;
     /** Flush the STT, causing it to process all pending text */
@@ -99,4 +120,5 @@ export declare abstract class SpeechStream implements AsyncIterableIterator<Spee
     close(): void;
     [Symbol.asyncIterator](): SpeechStream;
 }
+export {};
 //# sourceMappingURL=stt.d.ts.map

package/dist/stt/stt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/stt/stt.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,2CAA2C;AAC3C,oBAAY,eAAe;IACzB;;;;OAIG;IACH,eAAe,IAAI;IACnB;;OAEG;IACH,kBAAkB,IAAI;IACtB;;;OAGG;IACH,gBAAgB,IAAI;IACpB;;;OAGG;IACH,aAAa,IAAI;CAClB;AAED,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,sDAAsD;AACtD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;CAC9C;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;GAMG;AACH,8BAAsB,GAAG;;gBAGX,YAAY,EAAE,eAAe;IAIzC,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,8FAA8F;IAC9F,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAE5D;;;OAGG;IACH,QAAQ,CAAC,MAAM,IAAI,YAAY;CAChC;AAED;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,YAAW,qBAAqB,CAAC,WAAW,CAAC;IAC9E,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,KAAK,kCAAyC;IACxD,SAAS,CAAC,MAAM,UAAS;IAEzB,qCAAqC;IACrC,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAI5C,wDAAwD;IACxD,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,YAAY;CAGvC"}
+{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/stt/stt.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,2CAA2C;AAC3C,oBAAY,eAAe;IACzB;;;;OAIG;IACH,eAAe,IAAI;IACnB;;OAEG;IACH,kBAAkB,IAAI;IACtB;;;OAGG;IACH,gBAAgB,IAAI;IACpB;;;OAGG;IACH,aAAa,IAAI;IACjB,mEAAmE;IACnE,iBAAiB,IAAI;IACrB,iBAAiB,IAAI;CACtB;AAED,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,sDAAsD;AACtD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;CACpE,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IACtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAGX,YAAY,EAAE,eAAe;IAKzC,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,8FAA8F;IACxF,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAiBzD,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAEvE;;;OAGG;IACH,QAAQ,CAAC,MAAM,IAAI,YAAY;CAChC;AAED;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,YAAW,qBAAqB,CAAC,WAAW,CAAC;;IAC9E,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,MAAM,kCAAyC;IACzD,SAAS,CAAC,KAAK,kCAAyC;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,MAAM,UAAS;gBAGb,GAAG,EAAE,GAAG;cAKJ,cAAc;IAoB9B,qCAAqC;IACrC,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAI5C,wDAAwD;IACxD,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,YAAY;CAGvC"}

package/dist/stt/stt.js

@@ -1,26 +1,69 @@
+import { EventEmitter } from "node:events";
 import { AsyncIterableQueue } from "../utils.js";
 var SpeechEventType = /* @__PURE__ */ ((SpeechEventType2) => {
   SpeechEventType2[SpeechEventType2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
   SpeechEventType2[SpeechEventType2["INTERIM_TRANSCRIPT"] = 1] = "INTERIM_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["FINAL_TRANSCRIPT"] = 2] = "FINAL_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["END_OF_SPEECH"] = 3] = "END_OF_SPEECH";
+  SpeechEventType2[SpeechEventType2["RECOGNITION_USAGE"] = 4] = "RECOGNITION_USAGE";
+  SpeechEventType2[SpeechEventType2["METRICS_COLLECTED"] = 5] = "METRICS_COLLECTED";
   return SpeechEventType2;
 })(SpeechEventType || {});
-class STT {
+class STT extends EventEmitter {
   #capabilities;
   constructor(capabilities) {
+    super();
     this.#capabilities = capabilities;
   }
   /** Returns this STT's capabilities */
   get capabilities() {
     return this.#capabilities;
   }
+  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
+  async recognize(frame) {
+    const startTime = process.hrtime.bigint();
+    const event = await this._recognize(frame);
+    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1e6));
+    this.emit(5 /* METRICS_COLLECTED */, {
+      requestId: event.requestId ?? "",
+      timestamp: Date.now(),
+      duration,
+      label: this.label,
+      audioDuration: Array.isArray(frame) ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0) : frame.samplesPerChannel / frame.sampleRate,
+      streamed: false
+    });
+    return event;
+  }
 }
 class SpeechStream {
   static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
   input = new AsyncIterableQueue();
+  output = new AsyncIterableQueue();
   queue = new AsyncIterableQueue();
   closed = false;
+  #stt;
+  constructor(stt) {
+    this.#stt = stt;
+    this.monitorMetrics();
+  }
+  async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    for await (const event of this.queue) {
+      this.output.put(event);
+      if (event.type !== 4 /* RECOGNITION_USAGE */) continue;
+      const duration = process.hrtime.bigint() - startTime;
+      const metrics = {
+        timestamp: Date.now(),
+        requestId: event.requestId,
+        duration: Math.trunc(Number(duration / BigInt(1e6))),
+        label: this.label,
+        audioDuration: event.recognitionUsage.audioDuration,
+        streamed: true
+      };
+      this.#stt.emit(5 /* METRICS_COLLECTED */, metrics);
+    }
+    this.output.close();
+  }
   /** Push an audio frame to the STT */
   pushFrame(frame) {
     if (this.input.closed) {
@@ -52,12 +95,13 @@ class SpeechStream {
     this.input.close();
   }
   next() {
-    return this.queue.next();
+    return this.output.next();
   }
   /** Close both the input and output of the STT stream */
   close() {
     this.input.close();
     this.queue.close();
+    this.output.close();
     this.closed = true;
   }
   [Symbol.asyncIterator]() {

package/dist/stt/stt.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT {\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  abstract recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected closed = false;\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,0BAA0B;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AApBU,SAAAA;AAAA,GAAA;AAwDL,MAAe,IAAI;AAAA,EACxB;AAAA,EAEA,YAAY,cAA+B;AACzC,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAUF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAoE;AAAA,EAChF,QAAQ,IAAI,mBAAgC;AAAA,EAC5C,SAAS;AAAA;AAAA,EAGnB,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}
+{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { STTMetrics } from '../metrics/base.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  METRICS_COLLECTED = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport type STTCallbacks = {\n  [SpeechEventType.METRICS_COLLECTED]: (metrics: STTMetrics) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit(SpeechEventType.METRICS_COLLECTED, {\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      duration,\n      label: this.label,\n      audioDuration: Array.isArray(frame)\n        ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0)\n        : frame.samplesPerChannel / frame.sampleRate,\n      streamed: false,\n    });\n    return event;\n  }\n\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n\n  constructor(stt: STT) {\n    this.#stt = stt;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const duration = process.hrtime.bigint() - startTime;\n      const metrics: STTMetrics = {\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        duration: Math.trunc(Number(duration / BigInt(1000000))),\n        label: this.label,\n        audioDuration: event.recognitionUsage!.audioDuration,\n        streamed: true,\n      };\n      this.#stt.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    }\n    this.output.close();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAG7B,SAAS,0BAA0B;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AACA,EAAAA,kCAAA,uBAAoB,KAApB;AAvBU,SAAAA;AAAA,GAAA;AAqEL,MAAe,YAAa,aAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,WAAW,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AAC/E,SAAK,KAAK,2BAAmC;AAAA,MAC3C,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,eAAe,MAAM,QAAQ,KAAK,IAC9B,MAAM,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,oBAAoB,EAAE,YAAY,CAAC,IACpE,MAAM,oBAAoB,MAAM;AAAA,MACpC,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AASF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAoE;AAAA,EAChF,SAAS,IAAI,mBAAgC;AAAA,EAC7C,QAAQ,IAAI,mBAAgC;AAAA,EAE5C,SAAS;AAAA,EACnB;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AAExC,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,YAAM,UAAsB;AAAA,QAC1B,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,QACvD,OAAO,KAAK;AAAA,QACZ,eAAe,MAAM,iBAAkB;AAAA,QACvC,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,2BAAmC,OAAO;AAAA,IAC3D;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}

package/dist/tts/index.cjs

@@ -22,7 +22,8 @@ __export(tts_exports, {
   StreamAdapter: () => import_stream_adapter.StreamAdapter,
   StreamAdapterWrapper: () => import_stream_adapter.StreamAdapterWrapper,
   SynthesizeStream: () => import_tts.SynthesizeStream,
-  TTS: () => import_tts.TTS
+  TTS: () => import_tts.TTS,
+  TTSEvent: () => import_tts.TTSEvent
 });
 module.exports = __toCommonJS(tts_exports);
 var import_tts = require("./tts.cjs");
@@ -33,6 +34,7 @@ var import_stream_adapter = require("./stream_adapter.cjs");
   StreamAdapter,
   StreamAdapterWrapper,
   SynthesizeStream,
-  TTS
+  TTS,
+  TTSEvent
 });
 //# sourceMappingURL=index.cjs.map

package/dist/tts/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  type SynthesizedAudio,\n  type TTSCapabilities,\n  TTS,\n  SynthesizeStream,\n  ChunkedStream,\n} from './tts.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAMO;AACP,4BAAoD;","names":[]}
+{"version":3,"sources":["../../src/tts/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  type SynthesizedAudio,\n  type TTSCapabilities,\n  type TTSCallbacks,\n  TTS,\n  TTSEvent,\n  SynthesizeStream,\n  ChunkedStream,\n} from './tts.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAQO;AACP,4BAAoD;","names":[]}

package/dist/tts/index.d.ts

@@ -1,3 +1,3 @@
-export { type SynthesizedAudio, type TTSCapabilities, TTS, SynthesizeStream, ChunkedStream, } from './tts.js';
+export { type SynthesizedAudio, type TTSCapabilities, type TTSCallbacks, TTS, TTSEvent, SynthesizeStream, ChunkedStream, } from './tts.js';
 export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/tts/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tts/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,GAAG,EACH,gBAAgB,EAChB,aAAa,GACd,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tts/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,GAAG,EACH,QAAQ,EACR,gBAAgB,EAChB,aAAa,GACd,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/tts/index.js

@@ -1,5 +1,6 @@
 import {
   TTS,
+  TTSEvent,
   SynthesizeStream,
   ChunkedStream
 } from "./tts.js";
@@ -9,6 +10,7 @@ export {
   StreamAdapter,
   StreamAdapterWrapper,
   SynthesizeStream,
-  TTS
+  TTS,
+  TTSEvent
 };
 //# sourceMappingURL=index.js.map

package/dist/tts/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  type SynthesizedAudio,\n  type TTSCapabilities,\n  TTS,\n  SynthesizeStream,\n  ChunkedStream,\n} from './tts.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":"AAGA;AAAA,EAGE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe,4BAA4B;","names":[]}
+{"version":3,"sources":["../../src/tts/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  type SynthesizedAudio,\n  type TTSCapabilities,\n  type TTSCallbacks,\n  TTS,\n  TTSEvent,\n  SynthesizeStream,\n  ChunkedStream,\n} from './tts.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":"AAGA;AAAA,EAIE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe,4BAA4B;","names":[]}

package/dist/tts/stream_adapter.cjs

@@ -26,10 +26,16 @@ var import_tts = require("./tts.cjs");
 class StreamAdapter extends import_tts.TTS {
   #tts;
   #sentenceTokenizer;
+  label;
   constructor(tts, sentenceTokenizer) {
     super(tts.sampleRate, tts.numChannels, { streaming: true });
     this.#tts = tts;
     this.#sentenceTokenizer = sentenceTokenizer;
+    this.label = this.#tts.label;
+    this.label = `tts.StreamAdapter<${this.#tts.label}>`;
+    this.#tts.on(import_tts.TTSEvent.METRICS_COLLECTED, (metrics) => {
+      this.emit(import_tts.TTSEvent.METRICS_COLLECTED, metrics);
+    });
   }
   synthesize(text) {
     return this.#tts.synthesize(text);
@@ -41,12 +47,17 @@ class StreamAdapter extends import_tts.TTS {
 class StreamAdapterWrapper extends import_tts.SynthesizeStream {
   #tts;
   #sentenceStream;
+  label;
   constructor(tts, sentenceTokenizer) {
-    super();
+    super(tts);
     this.#tts = tts;
     this.#sentenceStream = sentenceTokenizer.stream();
+    this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;
     this.#run();
   }
+  async monitorMetrics() {
+    return;
+  }
   async #run() {
     const forwardInput = async () => {
       for await (const input of this.input) {
@@ -62,10 +73,10 @@ class StreamAdapterWrapper extends import_tts.SynthesizeStream {
     const synthesize = async () => {
       for await (const ev of this.#sentenceStream) {
         for await (const audio of this.#tts.synthesize(ev.token)) {
-          this.queue.put(audio);
+          this.output.put(audio);
         }
       }
-      this.queue.put(import_tts.SynthesizeStream.END_OF_STREAM);
+      this.output.put(import_tts.SynthesizeStream.END_OF_STREAM);
     };
     Promise.all([forwardInput(), synthesize()]);
   }

package/dist/tts/stream_adapter.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';\nimport type { ChunkedStream } from './tts.js';\nimport { SynthesizeStream, TTS } from './tts.js';\n\nexport class StreamAdapter extends TTS {\n  #tts: TTS;\n  #sentenceTokenizer: SentenceTokenizer;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts.sampleRate, tts.numChannels, { streaming: true });\n    this.#tts = tts;\n    this.#sentenceTokenizer = sentenceTokenizer;\n  }\n\n  synthesize(text: string): ChunkedStream {\n    return this.#tts.synthesize(text);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);\n  }\n}\n\nexport class StreamAdapterWrapper extends SynthesizeStream {\n  #tts: TTS;\n  #sentenceStream: SentenceStream;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super();\n    this.#tts = tts;\n    this.#sentenceStream = sentenceTokenizer.stream();\n\n    this.#run();\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#sentenceStream.flush();\n        } else {\n          this.#sentenceStream.pushText(input);\n        }\n      }\n      this.#sentenceStream.endInput();\n      this.#sentenceStream.close();\n    };\n\n    const synthesize = async () => {\n      for await (const ev of this.#sentenceStream) {\n        for await (const audio of this.#tts.synthesize(ev.token)) {\n          this.queue.put(audio);\n        }\n      }\n      this.queue.put(SynthesizeStream.END_OF_STREAM);\n    };\n\n    Promise.all([forwardInput(), synthesize()]);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,iBAAsC;AAE/B,MAAM,sBAAsB,eAAI;AAAA,EACrC;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,IAAI,YAAY,IAAI,aAAa,EAAE,WAAW,KAAK,CAAC;AAC1D,SAAK,OAAO;AACZ,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EAEA,WAAW,MAA6B;AACtC,WAAO,KAAK,KAAK,WAAW,IAAI;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,kBAAkB;AAAA,EACpE;AACF;AAEO,MAAM,6BAA6B,4BAAiB;AAAA,EACzD;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,kBAAkB,kBAAkB,OAAO;AAEhD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,4BAAiB,gBAAgB;AAC7C,eAAK,gBAAgB,MAAM;AAAA,QAC7B,OAAO;AACL,eAAK,gBAAgB,SAAS,KAAK;AAAA,QACrC;AAAA,MACF;AACA,WAAK,gBAAgB,SAAS;AAC9B,WAAK,gBAAgB,MAAM;AAAA,IAC7B;AAEA,UAAM,aAAa,YAAY;AAC7B,uBAAiB,MAAM,KAAK,iBAAiB;AAC3C,yBAAiB,SAAS,KAAK,KAAK,WAAW,GAAG,KAAK,GAAG;AACxD,eAAK,MAAM,IAAI,KAAK;AAAA,QACtB;AAAA,MACF;AACA,WAAK,MAAM,IAAI,4BAAiB,aAAa;AAAA,IAC/C;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,CAAC;AAAA,EAC5C;AACF;","names":[]}
+{"version":3,"sources":["../../src/tts/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';\nimport type { ChunkedStream } from './tts.js';\nimport { SynthesizeStream, TTS, TTSEvent } from './tts.js';\n\nexport class StreamAdapter extends TTS {\n  #tts: TTS;\n  #sentenceTokenizer: SentenceTokenizer;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts.sampleRate, tts.numChannels, { streaming: true });\n    this.#tts = tts;\n    this.#sentenceTokenizer = sentenceTokenizer;\n    this.label = this.#tts.label;\n    this.label = `tts.StreamAdapter<${this.#tts.label}>`;\n\n    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {\n      this.emit(TTSEvent.METRICS_COLLECTED, metrics);\n    });\n  }\n\n  synthesize(text: string): ChunkedStream {\n    return this.#tts.synthesize(text);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);\n  }\n}\n\nexport class StreamAdapterWrapper extends SynthesizeStream {\n  #tts: TTS;\n  #sentenceStream: SentenceStream;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts);\n    this.#tts = tts;\n    this.#sentenceStream = sentenceTokenizer.stream();\n    this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;\n\n    this.#run();\n  }\n\n  async monitorMetrics() {\n    return; // do nothing\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#sentenceStream.flush();\n        } else {\n          this.#sentenceStream.pushText(input);\n        }\n      }\n      this.#sentenceStream.endInput();\n      this.#sentenceStream.close();\n    };\n\n    const synthesize = async () => {\n      for await (const ev of this.#sentenceStream) {\n        for await (const audio of this.#tts.synthesize(ev.token)) {\n          this.output.put(audio);\n        }\n      }\n      this.output.put(SynthesizeStream.END_OF_STREAM);\n    };\n\n    Promise.all([forwardInput(), synthesize()]);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,iBAAgD;AAEzC,MAAM,sBAAsB,eAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,IAAI,YAAY,IAAI,aAAa,EAAE,WAAW,KAAK,CAAC;AAC1D,SAAK,OAAO;AACZ,SAAK,qBAAqB;AAC1B,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,oBAAS,mBAAmB,CAAC,YAAY;AACpD,WAAK,KAAK,oBAAS,mBAAmB,OAAO;AAAA,IAC/C,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,MAA6B;AACtC,WAAO,KAAK,KAAK,WAAW,IAAI;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,kBAAkB;AAAA,EACpE;AACF;AAEO,MAAM,6BAA6B,4BAAiB;AAAA,EACzD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,kBAAkB,kBAAkB,OAAO;AAChD,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAExD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,iBAAiB;AACrB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,4BAAiB,gBAAgB;AAC7C,eAAK,gBAAgB,MAAM;AAAA,QAC7B,OAAO;AACL,eAAK,gBAAgB,SAAS,KAAK;AAAA,QACrC;AAAA,MACF;AACA,WAAK,gBAAgB,SAAS;AAC9B,WAAK,gBAAgB,MAAM;AAAA,IAC7B;AAEA,UAAM,aAAa,YAAY;AAC7B,uBAAiB,MAAM,KAAK,iBAAiB;AAC3C,yBAAiB,SAAS,KAAK,KAAK,WAAW,GAAG,KAAK,GAAG;AACxD,eAAK,OAAO,IAAI,KAAK;AAAA,QACvB;AAAA,MACF;AACA,WAAK,OAAO,IAAI,4BAAiB,aAAa;AAAA,IAChD;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,CAAC;AAAA,EAC5C;AACF;","names":[]}

package/dist/tts/stream_adapter.d.ts

@@ -3,12 +3,15 @@ import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
 export declare class StreamAdapter extends TTS {
     #private;
+    label: string;
     constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
     synthesize(text: string): ChunkedStream;
     stream(): StreamAdapterWrapper;
 }
 export declare class StreamAdapterWrapper extends SynthesizeStream {
     #private;
+    label: string;
     constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
+    monitorMetrics(): Promise<void>;
 }
 //# sourceMappingURL=stream_adapter.d.ts.map

package/dist/tts/stream_adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/tts/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEjD,qBAAa,aAAc,SAAQ,GAAG;;gBAIxB,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;IAM1D,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAIvC,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,gBAAgB;;gBAI5C,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;CAgC3D"}
+{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/tts/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAY,MAAM,UAAU,CAAC;AAE3D,qBAAa,aAAc,SAAQ,GAAG;;IAGpC,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;IAY1D,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAIvC,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,gBAAgB;;IAGxD,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;IASpD,cAAc;CA4BrB"}