@livekit/agents 0.5.1 → 0.6.0

package/dist/vad.cjs

@@ -23,16 +23,19 @@ __export(vad_exports, {
   VADStream: () => VADStream
 });
 module.exports = __toCommonJS(vad_exports);
+var import_node_events = require("node:events");
 var import_utils = require("./utils.cjs");
 var VADEventType = /* @__PURE__ */ ((VADEventType2) => {
   VADEventType2[VADEventType2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
   VADEventType2[VADEventType2["INFERENCE_DONE"] = 1] = "INFERENCE_DONE";
   VADEventType2[VADEventType2["END_OF_SPEECH"] = 2] = "END_OF_SPEECH";
+  VADEventType2[VADEventType2["METRICS_COLLECTED"] = 3] = "METRICS_COLLECTED";
   return VADEventType2;
 })(VADEventType || {});
-class VAD {
+class VAD extends import_node_events.EventEmitter {
   #capabilities;
   constructor(capabilities) {
+    super();
     this.#capabilities = capabilities;
   }
   get capabilities() {
@@ -43,7 +46,44 @@ class VADStream {
   static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
   input = new import_utils.AsyncIterableQueue();
   queue = new import_utils.AsyncIterableQueue();
+  output = new import_utils.AsyncIterableQueue();
   closed = false;
+  #vad;
+  #lastActivityTime = BigInt(0);
+  constructor(vad) {
+    this.#vad = vad;
+    this.monitorMetrics();
+  }
+  async monitorMetrics() {
+    let inferenceDurationTotal = 0;
+    let inferenceCount = 0;
+    for await (const event of this.queue) {
+      this.output.put(event);
+      switch (event.type) {
+        case 0 /* START_OF_SPEECH */:
+          inferenceCount++;
+          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {
+            this.#vad.emit(3 /* METRICS_COLLECTED */, {
+              timestamp: Date.now(),
+              idleTime: Math.trunc(
+                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1e6))
+              ),
+              inferenceDurationTotal,
+              inferenceCount,
+              label: this.#vad.label
+            });
+            inferenceCount = 0;
+            inferenceDurationTotal = 0;
+          }
+          break;
+        case 1 /* INFERENCE_DONE */:
+        case 2 /* END_OF_SPEECH */:
+          this.#lastActivityTime = process.hrtime.bigint();
+          break;
+      }
+    }
+    this.output.close();
+  }
   pushFrame(frame) {
     if (this.input.closed) {
       throw new Error("Input is closed");
@@ -72,11 +112,12 @@ class VADStream {
     this.input.close();
   }
   next() {
-    return this.queue.next();
+    return this.output.next();
   }
   close() {
     this.input.close();
     this.queue.close();
+    this.output.close();
     this.closed = true;
   }
   [Symbol.asyncIterator]() {

package/dist/vad.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncIterableQueue } from './utils.js';\n\nexport enum VADEventType {\n  START_OF_SPEECH,\n  INFERENCE_DONE,\n  END_OF_SPEECH,\n}\n\nexport interface VADEvent {\n  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n  type: VADEventType;\n  /**\n   * Index of the audio sample where the event occurred, relative to the inference sample rate.\n   */\n  samplesIndex: number;\n  /** Timestamp when the event was fired. */\n  timestamp: number;\n  /** Duration of the detected speech segment in seconds. */\n  speechDuration: number;\n  /** Duration of the silence segment preceding or following the speech, in seconds. */\n  silenceDuration: number;\n  /**\n   * List of audio frames associated with the speech.\n   *\n   * @remarks\n   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n   * - For `inference_done` events, this contains the audio chunks that were processed.\n   * - For `end_of_speech` events, this contains the complete user speech.\n   */\n  frames: AudioFrame[];\n  /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n  probability: number;\n  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n  inferenceDuration: number;\n  /** Indicates whether speech was detected in the frames. */\n  speaking: boolean;\n}\n\nexport interface VADCapabilities {\n  updateInterval: number;\n}\n\nexport abstract class VAD {\n  #capabilities: VADCapabilities;\n  constructor(capabilities: VADCapabilities) {\n    this.#capabilities = capabilities;\n  }\n\n  get capabilities(): VADCapabilities {\n    return this.#capabilities;\n  }\n\n  /**\n   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n   */\n  abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<VADEvent>();\n  protected closed = false;\n\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(VADStream.FLUSH_SENTINEL);\n  }\n\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<VADEvent>> {\n    return this.queue.next();\n  }\n\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): VADStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAmC;AAE5B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAHU,SAAAA;AAAA,GAAA;AAwCL,MAAe,IAAI;AAAA,EACxB;AAAA,EACA,YAAY,cAA+B;AACzC,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAiE;AAAA,EAC7E,QAAQ,IAAI,gCAA6B;AAAA,EACzC,SAAS;AAAA,EAEnB,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,UAAU,cAAc;AAAA,EACzC;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0C;AACxC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA,EAEA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
+{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { VADMetrics } from './metrics/base.js';\nimport { AsyncIterableQueue } from './utils.js';\n\nexport enum VADEventType {\n  START_OF_SPEECH,\n  INFERENCE_DONE,\n  END_OF_SPEECH,\n  METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n  type: VADEventType;\n  /**\n   * Index of the audio sample where the event occurred, relative to the inference sample rate.\n   */\n  samplesIndex: number;\n  /** Timestamp when the event was fired. */\n  timestamp: number;\n  /** Duration of the speech segment. */\n  speechDuration: number;\n  /** Duration of the silence segment. */\n  silenceDuration: number;\n  /**\n   * List of audio frames associated with the speech.\n   *\n   * @remarks\n   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n   * - For `inference_done` events, this contains the audio chunks that were processed.\n   * - For `end_of_speech` events, this contains the complete user speech.\n   */\n  frames: AudioFrame[];\n  /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n  probability: number;\n  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n  inferenceDuration: number;\n  /** Indicates whether speech was detected in the frames. */\n  speaking: boolean;\n  /** Threshold used to detect silence. */\n  rawAccumulatedSilence: number;\n  /** Threshold used to detect speech. */\n  rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n  updateInterval: number;\n}\n\nexport type VADCallbacks = {\n  [VADEventType.METRICS_COLLECTED]: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n  #capabilities: VADCapabilities;\n  abstract label: string;\n\n  constructor(capabilities: VADCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  get capabilities(): VADCapabilities {\n    return this.#capabilities;\n  }\n\n  /**\n   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n   */\n  abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<VADEvent>();\n  protected output = new AsyncIterableQueue<VADEvent>();\n  protected closed = false;\n  #vad: VAD;\n  #lastActivityTime = BigInt(0);\n\n  constructor(vad: VAD) {\n    this.#vad = vad;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    let inferenceDurationTotal = 0;\n    let inferenceCount = 0;\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      switch (event.type) {\n        case VADEventType.START_OF_SPEECH:\n          inferenceCount++;\n          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n            this.#vad.emit(VADEventType.METRICS_COLLECTED, {\n              timestamp: Date.now(),\n              idleTime: Math.trunc(\n                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n              ),\n              inferenceDurationTotal,\n              inferenceCount,\n              label: this.#vad.label,\n            });\n\n            inferenceCount = 0;\n            inferenceDurationTotal = 0;\n          }\n          break;\n        case VADEventType.INFERENCE_DONE:\n        case VADEventType.END_OF_SPEECH:\n          this.#lastActivityTime = process.hrtime.bigint();\n          break;\n      }\n    }\n    this.output.close();\n  }\n\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(VADStream.FLUSH_SENTINEL);\n  }\n\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<VADEvent>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): VADStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,mBAAmC;AAE5B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAiE;AAAA,EAC7E,QAAQ,IAAI,gCAA6B;AAAA,EACzC,SAAS,IAAI,gCAA6B;AAAA,EAC1C,SAAS;AAAA,EACnB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EAE5B,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,yBAAyB;AAC7B,QAAI,iBAAiB;AAErB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,2BAAgC;AAAA,cAC7C,WAAW,KAAK,IAAI;AAAA,cACpB,UAAU,KAAK;AAAA,gBACb,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,qCAAyB;AAAA,UAC3B;AACA;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAEA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,UAAU,cAAc;AAAA,EACzC;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0C;AACxC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}

package/dist/vad.d.ts

@@ -1,9 +1,12 @@
 import type { AudioFrame } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { VADMetrics } from './metrics/base.js';
 import { AsyncIterableQueue } from './utils.js';
 export declare enum VADEventType {
     START_OF_SPEECH = 0,
     INFERENCE_DONE = 1,
-    END_OF_SPEECH = 2
+    END_OF_SPEECH = 2,
+    METRICS_COLLECTED = 3
 }
 export interface VADEvent {
     /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */
@@ -14,9 +17,9 @@ export interface VADEvent {
     samplesIndex: number;
     /** Timestamp when the event was fired. */
     timestamp: number;
-    /** Duration of the detected speech segment in seconds. */
+    /** Duration of the speech segment. */
     speechDuration: number;
-    /** Duration of the silence segment preceding or following the speech, in seconds. */
+    /** Duration of the silence segment. */
     silenceDuration: number;
     /**
      * List of audio frames associated with the speech.
@@ -33,12 +36,21 @@ export interface VADEvent {
     inferenceDuration: number;
     /** Indicates whether speech was detected in the frames. */
     speaking: boolean;
+    /** Threshold used to detect silence. */
+    rawAccumulatedSilence: number;
+    /** Threshold used to detect speech. */
+    rawAccumulatedSpeech: number;
 }
 export interface VADCapabilities {
     updateInterval: number;
 }
-export declare abstract class VAD {
+export type VADCallbacks = {
+    [VADEventType.METRICS_COLLECTED]: (metrics: VADMetrics) => void;
+};
+declare const VAD_base: new () => TypedEmitter<VADCallbacks>;
+export declare abstract class VAD extends VAD_base {
     #private;
+    abstract label: string;
     constructor(capabilities: VADCapabilities);
     get capabilities(): VADCapabilities;
     /**
@@ -47,10 +59,14 @@ export declare abstract class VAD {
     abstract stream(): VADStream;
 }
 export declare abstract class VADStream implements AsyncIterableIterator<VADEvent> {
+    #private;
     protected static readonly FLUSH_SENTINEL: unique symbol;
     protected input: AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
     protected queue: AsyncIterableQueue<VADEvent>;
+    protected output: AsyncIterableQueue<VADEvent>;
     protected closed: boolean;
+    constructor(vad: VAD);
+    protected monitorMetrics(): Promise<void>;
     pushFrame(frame: AudioFrame): void;
     flush(): void;
     endInput(): void;
@@ -58,4 +74,5 @@ export declare abstract class VADStream implements AsyncIterableIterator<VADEven
     close(): void;
     [Symbol.asyncIterator](): VADStream;
 }
+export {};
 //# sourceMappingURL=vad.d.ts.map

package/dist/vad.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,oBAAY,YAAY;IACtB,eAAe,IAAA;IACf,cAAc,IAAA;IACd,aAAa,IAAA;CACd;AAED,MAAM,WAAW,QAAQ;IACvB,oFAAoF;IACpF,IAAI,EAAE,YAAY,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,cAAc,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,6EAA6E;IAC7E,WAAW,EAAE,MAAM,CAAC;IACpB,0FAA0F;IAC1F,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,8BAAsB,GAAG;;gBAEX,YAAY,EAAE,eAAe;IAIzC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,SAAS;CAC7B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,QAAQ,CAAC;IACxE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,mEAA0E;IACzF,SAAS,CAAC,KAAK,+BAAsC;IACrD,SAAS,CAAC,MAAM,UAAS;IAEzB,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,KAAK;IAUL,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAIzC,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}
+{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,oBAAY,YAAY;IACtB,eAAe,IAAA;IACf,cAAc,IAAA;IACd,aAAa,IAAA;IACb,iBAAiB,IAAA;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,oFAAoF;IACpF,IAAI,EAAE,YAAY,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,6EAA6E;IAC7E,WAAW,EAAE,MAAM,CAAC;IACpB,0FAA0F;IAC1F,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,QAAQ,EAAE,OAAO,CAAC;IAClB,wCAAwC;IACxC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uCAAuC;IACvC,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;CACjE,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAEtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,YAAY,EAAE,eAAe;IAKzC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,SAAS;CAC7B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,QAAQ,CAAC;;IACxE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,mEAA0E;IACzF,SAAS,CAAC,KAAK,+BAAsC;IACrD,SAAS,CAAC,MAAM,+BAAsC;IACtD,SAAS,CAAC,MAAM,UAAS;gBAIb,GAAG,EAAE,GAAG;cAKJ,cAAc;IAiC9B,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,KAAK;IAUL,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAIzC,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}

package/dist/vad.js

@@ -1,13 +1,16 @@
+import { EventEmitter } from "node:events";
 import { AsyncIterableQueue } from "./utils.js";
 var VADEventType = /* @__PURE__ */ ((VADEventType2) => {
   VADEventType2[VADEventType2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
   VADEventType2[VADEventType2["INFERENCE_DONE"] = 1] = "INFERENCE_DONE";
   VADEventType2[VADEventType2["END_OF_SPEECH"] = 2] = "END_OF_SPEECH";
+  VADEventType2[VADEventType2["METRICS_COLLECTED"] = 3] = "METRICS_COLLECTED";
   return VADEventType2;
 })(VADEventType || {});
-class VAD {
+class VAD extends EventEmitter {
   #capabilities;
   constructor(capabilities) {
+    super();
     this.#capabilities = capabilities;
   }
   get capabilities() {
@@ -18,7 +21,44 @@ class VADStream {
   static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
   input = new AsyncIterableQueue();
   queue = new AsyncIterableQueue();
+  output = new AsyncIterableQueue();
   closed = false;
+  #vad;
+  #lastActivityTime = BigInt(0);
+  constructor(vad) {
+    this.#vad = vad;
+    this.monitorMetrics();
+  }
+  async monitorMetrics() {
+    let inferenceDurationTotal = 0;
+    let inferenceCount = 0;
+    for await (const event of this.queue) {
+      this.output.put(event);
+      switch (event.type) {
+        case 0 /* START_OF_SPEECH */:
+          inferenceCount++;
+          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {
+            this.#vad.emit(3 /* METRICS_COLLECTED */, {
+              timestamp: Date.now(),
+              idleTime: Math.trunc(
+                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1e6))
+              ),
+              inferenceDurationTotal,
+              inferenceCount,
+              label: this.#vad.label
+            });
+            inferenceCount = 0;
+            inferenceDurationTotal = 0;
+          }
+          break;
+        case 1 /* INFERENCE_DONE */:
+        case 2 /* END_OF_SPEECH */:
+          this.#lastActivityTime = process.hrtime.bigint();
+          break;
+      }
+    }
+    this.output.close();
+  }
   pushFrame(frame) {
     if (this.input.closed) {
       throw new Error("Input is closed");
@@ -47,11 +87,12 @@ class VADStream {
     this.input.close();
   }
   next() {
-    return this.queue.next();
+    return this.output.next();
   }
   close() {
     this.input.close();
     this.queue.close();
+    this.output.close();
     this.closed = true;
   }
   [Symbol.asyncIterator]() {

package/dist/vad.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncIterableQueue } from './utils.js';\n\nexport enum VADEventType {\n  START_OF_SPEECH,\n  INFERENCE_DONE,\n  END_OF_SPEECH,\n}\n\nexport interface VADEvent {\n  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n  type: VADEventType;\n  /**\n   * Index of the audio sample where the event occurred, relative to the inference sample rate.\n   */\n  samplesIndex: number;\n  /** Timestamp when the event was fired. */\n  timestamp: number;\n  /** Duration of the detected speech segment in seconds. */\n  speechDuration: number;\n  /** Duration of the silence segment preceding or following the speech, in seconds. */\n  silenceDuration: number;\n  /**\n   * List of audio frames associated with the speech.\n   *\n   * @remarks\n   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n   * - For `inference_done` events, this contains the audio chunks that were processed.\n   * - For `end_of_speech` events, this contains the complete user speech.\n   */\n  frames: AudioFrame[];\n  /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n  probability: number;\n  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n  inferenceDuration: number;\n  /** Indicates whether speech was detected in the frames. */\n  speaking: boolean;\n}\n\nexport interface VADCapabilities {\n  updateInterval: number;\n}\n\nexport abstract class VAD {\n  #capabilities: VADCapabilities;\n  constructor(capabilities: VADCapabilities) {\n    this.#capabilities = capabilities;\n  }\n\n  get capabilities(): VADCapabilities {\n    return this.#capabilities;\n  }\n\n  /**\n   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n   */\n  abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<VADEvent>();\n  protected closed = false;\n\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(VADStream.FLUSH_SENTINEL);\n  }\n\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<VADEvent>> {\n    return this.queue.next();\n  }\n\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): VADStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,0BAA0B;AAE5B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAHU,SAAAA;AAAA,GAAA;AAwCL,MAAe,IAAI;AAAA,EACxB;AAAA,EACA,YAAY,cAA+B;AACzC,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAiE;AAAA,EAC7E,QAAQ,IAAI,mBAA6B;AAAA,EACzC,SAAS;AAAA,EAEnB,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,UAAU,cAAc;AAAA,EACzC;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0C;AACxC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA,EAEA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
+{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { VADMetrics } from './metrics/base.js';\nimport { AsyncIterableQueue } from './utils.js';\n\nexport enum VADEventType {\n  START_OF_SPEECH,\n  INFERENCE_DONE,\n  END_OF_SPEECH,\n  METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n  type: VADEventType;\n  /**\n   * Index of the audio sample where the event occurred, relative to the inference sample rate.\n   */\n  samplesIndex: number;\n  /** Timestamp when the event was fired. */\n  timestamp: number;\n  /** Duration of the speech segment. */\n  speechDuration: number;\n  /** Duration of the silence segment. */\n  silenceDuration: number;\n  /**\n   * List of audio frames associated with the speech.\n   *\n   * @remarks\n   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n   * - For `inference_done` events, this contains the audio chunks that were processed.\n   * - For `end_of_speech` events, this contains the complete user speech.\n   */\n  frames: AudioFrame[];\n  /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n  probability: number;\n  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n  inferenceDuration: number;\n  /** Indicates whether speech was detected in the frames. */\n  speaking: boolean;\n  /** Threshold used to detect silence. */\n  rawAccumulatedSilence: number;\n  /** Threshold used to detect speech. */\n  rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n  updateInterval: number;\n}\n\nexport type VADCallbacks = {\n  [VADEventType.METRICS_COLLECTED]: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n  #capabilities: VADCapabilities;\n  abstract label: string;\n\n  constructor(capabilities: VADCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  get capabilities(): VADCapabilities {\n    return this.#capabilities;\n  }\n\n  /**\n   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n   */\n  abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<VADEvent>();\n  protected output = new AsyncIterableQueue<VADEvent>();\n  protected closed = false;\n  #vad: VAD;\n  #lastActivityTime = BigInt(0);\n\n  constructor(vad: VAD) {\n    this.#vad = vad;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    let inferenceDurationTotal = 0;\n    let inferenceCount = 0;\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      switch (event.type) {\n        case VADEventType.START_OF_SPEECH:\n          inferenceCount++;\n          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n            this.#vad.emit(VADEventType.METRICS_COLLECTED, {\n              timestamp: Date.now(),\n              idleTime: Math.trunc(\n                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n              ),\n              inferenceDurationTotal,\n              inferenceCount,\n              label: this.#vad.label,\n            });\n\n            inferenceCount = 0;\n            inferenceDurationTotal = 0;\n          }\n          break;\n        case VADEventType.INFERENCE_DONE:\n        case VADEventType.END_OF_SPEECH:\n          this.#lastActivityTime = process.hrtime.bigint();\n          break;\n      }\n    }\n    this.output.close();\n  }\n\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(VADStream.FLUSH_SENTINEL);\n  }\n\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<VADEvent>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): VADStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,0BAA0B;AAE5B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAiE;AAAA,EAC7E,QAAQ,IAAI,mBAA6B;AAAA,EACzC,SAAS,IAAI,mBAA6B;AAAA,EAC1C,SAAS;AAAA,EACnB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EAE5B,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,yBAAyB;AAC7B,QAAI,iBAAiB;AAErB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,2BAAgC;AAAA,cAC7C,WAAW,KAAK,IAAI;AAAA,cACpB,UAAU,KAAK;AAAA,gBACb,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,qCAAyB;AAAA,UAC3B;AACA;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAEA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,UAAU,cAAc;AAAA,EACzC;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0C;AACxC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

package/src/index.ts

@@ -11,6 +11,7 @@
  */
 import * as cli from './cli.js';
 import * as llm from './llm/index.js';
+import * as metrics from './metrics/index.js';
 import * as multimodal from './multimodal/index.js';
 import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
@@ -28,4 +29,4 @@ export * from './generator.js';
 export * from './audio.js';
 export * from './transcription.js';
 
-export { cli, stt, tts, llm, pipeline, multimodal, tokenize };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize, metrics };

package/src/llm/index.ts

@@ -25,6 +25,8 @@ export {
   type CompletionUsage,
   type Choice,
   type ChatChunk,
+  type LLMCallbacks,
+  LLMEvent,
   LLM,
   LLMStream,
 } from './llm.js';

package/src/llm/llm.ts

@@ -1,6 +1,9 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { EventEmitter } from 'node:events';
+import type { LLMMetrics } from '../metrics/base.js';
 import { AsyncIterableQueue } from '../utils.js';
 import type { ChatContext, ChatRole } from './chat_context.js';
 import type { FunctionCallInfo, FunctionContext } from './function_context.js';
@@ -28,7 +31,15 @@ export interface ChatChunk {
   usage?: CompletionUsage;
 }
 
-export abstract class LLM {
+export enum LLMEvent {
+  METRICS_COLLECTED,
+}
+
+export type LLMCallbacks = {
+  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+};
+
+export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
@@ -48,16 +59,56 @@ export abstract class LLM {
 }
 
 export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+  protected output = new AsyncIterableQueue<ChatChunk>();
   protected queue = new AsyncIterableQueue<ChatChunk>();
   protected closed = false;
   protected _functionCalls: FunctionCallInfo[] = [];
+  abstract label: string;
 
+  #llm: LLM;
   #chatCtx: ChatContext;
   #fncCtx?: FunctionContext;
 
-  constructor(chatCtx: ChatContext, fncCtx?: FunctionContext) {
+  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {
+    this.#llm = llm;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.monitorMetrics();
+  }
+
+  protected async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    let ttft: bigint | undefined;
+    let requestId = '';
+    let usage: CompletionUsage | undefined;
+
+    for await (const ev of this.queue) {
+      this.output.put(ev);
+      requestId = ev.requestId;
+      if (!ttft) {
+        ttft = process.hrtime.bigint() - startTime;
+      }
+      if (ev.usage) {
+        usage = ev.usage;
+      }
+    }
+    this.output.close();
+
+    const duration = process.hrtime.bigint() - startTime;
+    const metrics: LLMMetrics = {
+      timestamp: Date.now(),
+      requestId,
+      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),
+      duration: Math.trunc(Number(duration / BigInt(1000000))),
+      cancelled: false, // XXX(nbsp)
+      label: this.label,
+      completionTokens: usage?.completionTokens || 0,
+      promptTokens: usage?.promptTokens || 0,
+      totalTokens: usage?.totalTokens || 0,
+      tokensPerSecond:
+        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),
+    };
+    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);
   }
 
   /** List of called functions from this stream. */
@@ -88,10 +139,11 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   next(): Promise<IteratorResult<ChatChunk>> {
-    return this.queue.next();
+    return this.output.next();
   }
 
   close() {
+    this.output.close();
     this.queue.close();
     this.closed = true;
   }

package/src/metrics/base.ts

@@ -0,0 +1,127 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export interface LLMMetrics {
+  requestId: string;
+  timestamp: number;
+  ttft: number;
+  duration: number;
+  label: string;
+  cancelled: boolean;
+  completionTokens: number;
+  promptTokens: number;
+  totalTokens: number;
+  tokensPerSecond: number;
+  error?: Error;
+}
+
+export interface STTMetrics {
+  requestId: string;
+  timestamp: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface TTSMetrics {
+  requestId: string;
+  timestamp: number;
+  ttfb: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  cancelled: boolean;
+  charactersCount: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface VADMetrics {
+  timestamp: number;
+  idleTime: number;
+  inferenceDurationTotal: number;
+  inferenceCount: number;
+  label: string;
+}
+
+export interface PipelineEOUMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+  /** Timestamp of when the event was recorded */
+  timestamp: number;
+  /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+  endOfUtteranceDelay: number;
+  /**
+   * Time taken to obtain the transcript after the end of the user's speech.
+   *
+   * @remarks
+   * May be 0 if the transcript was already available.
+   */
+  transcriptionDelay: number;
+}
+
+export interface PipelineLLMMetrics extends LLMMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export interface PipelineTTSMetrics extends TTSMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+
+export class MultimodalLLMError extends Error {
+  type?: string;
+  reason?: string;
+  code?: string;
+  constructor(
+    {
+      type,
+      reason,
+      code,
+      message,
+    }: { type?: string; reason?: string; code?: string; message?: string } = {},
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.type = type;
+    this.reason = reason;
+    this.code = code;
+  }
+}
+
+export interface MultimodalLLMMetrics extends LLMMetrics {
+  inputTokenDetails: {
+    cachedTokens: number;
+    textTokens: number;
+    audioTokens: number;
+  };
+  outputTokenDetails: {
+    textTokens: number;
+    audioTokens: number;
+  };
+}
+
+export type AgentMetrics =
+  | STTMetrics
+  | LLMMetrics
+  | TTSMetrics
+  | VADMetrics
+  | PipelineSTTMetrics
+  | PipelineEOUMetrics
+  | PipelineLLMMetrics
+  | PipelineTTSMetrics
+  | PipelineVADMetrics
+  | MultimodalLLMMetrics;

package/src/metrics/index.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type {
+  AgentMetrics,
+  STTMetrics,
+  LLMMetrics,
+  TTSMetrics,
+  VADMetrics,
+  PipelineSTTMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  PipelineVADMetrics,
+  MultimodalLLMMetrics,
+} from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';

package/src/metrics/usage_collector.ts

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AgentMetrics } from './base.js';
+import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';
+
+export interface UsageSummary {
+  llmPromptTokens: number;
+  llmCompletionTokens: number;
+  ttsCharactersCount: number;
+  sttAudioDuration: number;
+}
+
+export class UsageCollector {
+  #summary: UsageSummary;
+
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0,
+    };
+  }
+
+  collect(metrics: AgentMetrics) {
+    if (isLLMMetrics(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (isTTSMetrics(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (isSTTMetrics(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+
+  get summary(): UsageSummary {
+    return { ...this.#summary };
+  }
+}