@livekit/agents 0.5.2 → 0.6.0

package/dist/vad.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncIterableQueue } from './utils.js';\n\nexport enum VADEventType {\n  START_OF_SPEECH,\n  INFERENCE_DONE,\n  END_OF_SPEECH,\n}\n\nexport interface VADEvent {\n  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n  type: VADEventType;\n  /**\n   * Index of the audio sample where the event occurred, relative to the inference sample rate.\n   */\n  samplesIndex: number;\n  /** Timestamp when the event was fired. */\n  timestamp: number;\n  /** Duration of the detected speech segment in seconds. */\n  speechDuration: number;\n  /** Duration of the silence segment preceding or following the speech, in seconds. */\n  silenceDuration: number;\n  /**\n   * List of audio frames associated with the speech.\n   *\n   * @remarks\n   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n   * - For `inference_done` events, this contains the audio chunks that were processed.\n   * - For `end_of_speech` events, this contains the complete user speech.\n   */\n  frames: AudioFrame[];\n  /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n  probability: number;\n  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n  inferenceDuration: number;\n  /** Indicates whether speech was detected in the frames. */\n  speaking: boolean;\n}\n\nexport interface VADCapabilities {\n  updateInterval: number;\n}\n\nexport abstract class VAD {\n  #capabilities: VADCapabilities;\n  constructor(capabilities: VADCapabilities) {\n    this.#capabilities = capabilities;\n  }\n\n  get capabilities(): VADCapabilities {\n    return this.#capabilities;\n  }\n\n  /**\n   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n   */\n  abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<VADEvent>();\n  protected closed = false;\n\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(VADStream.FLUSH_SENTINEL);\n  }\n\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<VADEvent>> {\n    return this.queue.next();\n  }\n\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): VADStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,0BAA0B;AAE5B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAHU,SAAAA;AAAA,GAAA;AAwCL,MAAe,IAAI;AAAA,EACxB;AAAA,EACA,YAAY,cAA+B;AACzC,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAiE;AAAA,EAC7E,QAAQ,IAAI,mBAA6B;AAAA,EACzC,SAAS;AAAA,EAEnB,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,UAAU,cAAc;AAAA,EACzC;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0C;AACxC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA,EAEA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
+{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { VADMetrics } from './metrics/base.js';\nimport { AsyncIterableQueue } from './utils.js';\n\nexport enum VADEventType {\n  START_OF_SPEECH,\n  INFERENCE_DONE,\n  END_OF_SPEECH,\n  METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n  type: VADEventType;\n  /**\n   * Index of the audio sample where the event occurred, relative to the inference sample rate.\n   */\n  samplesIndex: number;\n  /** Timestamp when the event was fired. */\n  timestamp: number;\n  /** Duration of the speech segment. */\n  speechDuration: number;\n  /** Duration of the silence segment. */\n  silenceDuration: number;\n  /**\n   * List of audio frames associated with the speech.\n   *\n   * @remarks\n   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n   * - For `inference_done` events, this contains the audio chunks that were processed.\n   * - For `end_of_speech` events, this contains the complete user speech.\n   */\n  frames: AudioFrame[];\n  /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n  probability: number;\n  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n  inferenceDuration: number;\n  /** Indicates whether speech was detected in the frames. */\n  speaking: boolean;\n  /** Threshold used to detect silence. */\n  rawAccumulatedSilence: number;\n  /** Threshold used to detect speech. */\n  rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n  updateInterval: number;\n}\n\nexport type VADCallbacks = {\n  [VADEventType.METRICS_COLLECTED]: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n  #capabilities: VADCapabilities;\n  abstract label: string;\n\n  constructor(capabilities: VADCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  get capabilities(): VADCapabilities {\n    return this.#capabilities;\n  }\n\n  /**\n   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n   */\n  abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<VADEvent>();\n  protected output = new AsyncIterableQueue<VADEvent>();\n  protected closed = false;\n  #vad: VAD;\n  #lastActivityTime = BigInt(0);\n\n  constructor(vad: VAD) {\n    this.#vad = vad;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    let inferenceDurationTotal = 0;\n    let inferenceCount = 0;\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      switch (event.type) {\n        case VADEventType.START_OF_SPEECH:\n          inferenceCount++;\n          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n            this.#vad.emit(VADEventType.METRICS_COLLECTED, {\n              timestamp: Date.now(),\n              idleTime: Math.trunc(\n                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n              ),\n              inferenceDurationTotal,\n              inferenceCount,\n              label: this.#vad.label,\n            });\n\n            inferenceCount = 0;\n            inferenceDurationTotal = 0;\n          }\n          break;\n        case VADEventType.INFERENCE_DONE:\n        case VADEventType.END_OF_SPEECH:\n          this.#lastActivityTime = process.hrtime.bigint();\n          break;\n      }\n    }\n    this.output.close();\n  }\n\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(VADStream.FLUSH_SENTINEL);\n  }\n\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<VADEvent>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): VADStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,0BAA0B;AAE5B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAiE;AAAA,EAC7E,QAAQ,IAAI,mBAA6B;AAAA,EACzC,SAAS,IAAI,mBAA6B;AAAA,EAC1C,SAAS;AAAA,EACnB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EAE5B,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,yBAAyB;AAC7B,QAAI,iBAAiB;AAErB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,2BAAgC;AAAA,cAC7C,WAAW,KAAK,IAAI;AAAA,cACpB,UAAU,KAAK;AAAA,gBACb,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,qCAAyB;AAAA,UAC3B;AACA;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAEA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,UAAU,cAAc;AAAA,EACzC;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0C;AACxC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

package/src/index.ts

@@ -11,6 +11,7 @@
  */
 import * as cli from './cli.js';
 import * as llm from './llm/index.js';
+import * as metrics from './metrics/index.js';
 import * as multimodal from './multimodal/index.js';
 import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
@@ -28,4 +29,4 @@ export * from './generator.js';
 export * from './audio.js';
 export * from './transcription.js';
 
-export { cli, stt, tts, llm, pipeline, multimodal, tokenize };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize, metrics };

package/src/llm/index.ts

@@ -25,6 +25,8 @@ export {
   type CompletionUsage,
   type Choice,
   type ChatChunk,
+  type LLMCallbacks,
+  LLMEvent,
   LLM,
   LLMStream,
 } from './llm.js';

package/src/llm/llm.ts

@@ -1,6 +1,9 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { EventEmitter } from 'node:events';
+import type { LLMMetrics } from '../metrics/base.js';
 import { AsyncIterableQueue } from '../utils.js';
 import type { ChatContext, ChatRole } from './chat_context.js';
 import type { FunctionCallInfo, FunctionContext } from './function_context.js';
@@ -28,7 +31,15 @@ export interface ChatChunk {
   usage?: CompletionUsage;
 }
 
-export abstract class LLM {
+export enum LLMEvent {
+  METRICS_COLLECTED,
+}
+
+export type LLMCallbacks = {
+  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+};
+
+export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
@@ -48,16 +59,56 @@ export abstract class LLM {
 }
 
 export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+  protected output = new AsyncIterableQueue<ChatChunk>();
   protected queue = new AsyncIterableQueue<ChatChunk>();
   protected closed = false;
   protected _functionCalls: FunctionCallInfo[] = [];
+  abstract label: string;
 
+  #llm: LLM;
   #chatCtx: ChatContext;
   #fncCtx?: FunctionContext;
 
-  constructor(chatCtx: ChatContext, fncCtx?: FunctionContext) {
+  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {
+    this.#llm = llm;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.monitorMetrics();
+  }
+
+  protected async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    let ttft: bigint | undefined;
+    let requestId = '';
+    let usage: CompletionUsage | undefined;
+
+    for await (const ev of this.queue) {
+      this.output.put(ev);
+      requestId = ev.requestId;
+      if (!ttft) {
+        ttft = process.hrtime.bigint() - startTime;
+      }
+      if (ev.usage) {
+        usage = ev.usage;
+      }
+    }
+    this.output.close();
+
+    const duration = process.hrtime.bigint() - startTime;
+    const metrics: LLMMetrics = {
+      timestamp: Date.now(),
+      requestId,
+      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),
+      duration: Math.trunc(Number(duration / BigInt(1000000))),
+      cancelled: false, // XXX(nbsp)
+      label: this.label,
+      completionTokens: usage?.completionTokens || 0,
+      promptTokens: usage?.promptTokens || 0,
+      totalTokens: usage?.totalTokens || 0,
+      tokensPerSecond:
+        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),
+    };
+    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);
   }
 
   /** List of called functions from this stream. */
@@ -88,10 +139,11 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   next(): Promise<IteratorResult<ChatChunk>> {
-    return this.queue.next();
+    return this.output.next();
   }
 
   close() {
+    this.output.close();
     this.queue.close();
     this.closed = true;
   }

package/src/metrics/base.ts

@@ -0,0 +1,127 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export interface LLMMetrics {
+  requestId: string;
+  timestamp: number;
+  ttft: number;
+  duration: number;
+  label: string;
+  cancelled: boolean;
+  completionTokens: number;
+  promptTokens: number;
+  totalTokens: number;
+  tokensPerSecond: number;
+  error?: Error;
+}
+
+export interface STTMetrics {
+  requestId: string;
+  timestamp: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface TTSMetrics {
+  requestId: string;
+  timestamp: number;
+  ttfb: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  cancelled: boolean;
+  charactersCount: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface VADMetrics {
+  timestamp: number;
+  idleTime: number;
+  inferenceDurationTotal: number;
+  inferenceCount: number;
+  label: string;
+}
+
+export interface PipelineEOUMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+  /** Timestamp of when the event was recorded */
+  timestamp: number;
+  /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+  endOfUtteranceDelay: number;
+  /**
+   * Time taken to obtain the transcript after the end of the user's speech.
+   *
+   * @remarks
+   * May be 0 if the transcript was already available.
+   */
+  transcriptionDelay: number;
+}
+
+export interface PipelineLLMMetrics extends LLMMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export interface PipelineTTSMetrics extends TTSMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+
+export class MultimodalLLMError extends Error {
+  type?: string;
+  reason?: string;
+  code?: string;
+  constructor(
+    {
+      type,
+      reason,
+      code,
+      message,
+    }: { type?: string; reason?: string; code?: string; message?: string } = {},
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.type = type;
+    this.reason = reason;
+    this.code = code;
+  }
+}
+
+export interface MultimodalLLMMetrics extends LLMMetrics {
+  inputTokenDetails: {
+    cachedTokens: number;
+    textTokens: number;
+    audioTokens: number;
+  };
+  outputTokenDetails: {
+    textTokens: number;
+    audioTokens: number;
+  };
+}
+
+export type AgentMetrics =
+  | STTMetrics
+  | LLMMetrics
+  | TTSMetrics
+  | VADMetrics
+  | PipelineSTTMetrics
+  | PipelineEOUMetrics
+  | PipelineLLMMetrics
+  | PipelineTTSMetrics
+  | PipelineVADMetrics
+  | MultimodalLLMMetrics;

package/src/metrics/index.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type {
+  AgentMetrics,
+  STTMetrics,
+  LLMMetrics,
+  TTSMetrics,
+  VADMetrics,
+  PipelineSTTMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  PipelineVADMetrics,
+  MultimodalLLMMetrics,
+} from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';

package/src/metrics/usage_collector.ts

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AgentMetrics } from './base.js';
+import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';
+
+export interface UsageSummary {
+  llmPromptTokens: number;
+  llmCompletionTokens: number;
+  ttsCharactersCount: number;
+  sttAudioDuration: number;
+}
+
+export class UsageCollector {
+  #summary: UsageSummary;
+
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0,
+    };
+  }
+
+  collect(metrics: AgentMetrics) {
+    if (isLLMMetrics(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (isTTSMetrics(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (isSTTMetrics(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+
+  get summary(): UsageSummary {
+    return { ...this.#summary };
+  }
+}

package/src/metrics/utils.ts

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../log.js';
+import type {
+  AgentMetrics,
+  LLMMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  STTMetrics,
+  TTSMetrics,
+  VADMetrics,
+} from './base.js';
+
+export const logMetrics = (metrics: AgentMetrics) => {
+  const logger = log();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        ttft: metrics.ttft,
+        inputTokens: metrics.promptTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: metrics.tokensPerSecond,
+      })
+      .info('Pipeline LLM metrics');
+  } else if (isLLMMetrics(metrics)) {
+    logger
+      .child({
+        ttft: metrics.ttft,
+        inputTokens: metrics.promptTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: metrics.tokensPerSecond,
+      })
+      .info('LLM metrics');
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        ttfb: metrics.ttfb,
+        audioDuration: metrics.audioDuration,
+      })
+      .info('Pipeline TTS metrics');
+  } else if (isTTSMetrics(metrics)) {
+    logger
+      .child({
+        ttfb: metrics.ttfb,
+        audioDuration: metrics.audioDuration,
+      })
+      .info('TTS metrics');
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+        transcriptionDelay: metrics.transcriptionDelay,
+      })
+      .info('Pipeline EOU metrics');
+  } else if (isSTTMetrics(metrics)) {
+    logger
+      .child({
+        audioDuration: metrics.audioDuration,
+      })
+      .info('STT metrics');
+  }
+};
+
+export const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {
+  return !!(metrics as LLMMetrics).ttft;
+};
+
+export const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {
+  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;
+};
+
+export const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {
+  return !!(metrics as VADMetrics).inferenceCount;
+};
+
+export const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {
+  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;
+};
+
+export const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {
+  return !!(metrics as TTSMetrics).ttfb;
+};
+
+export const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {
+  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;
+};
+
+export const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {
+  return !(
+    isLLMMetrics(metrics) ||
+    isVADMetrics(metrics) ||
+    isPipelineEOUMetrics(metrics) ||
+    isTTSMetrics(metrics)
+  );
+};

package/src/multimodal/multimodal_agent.ts

@@ -21,6 +21,7 @@ import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
 import * as llm from '../llm/index.js';
 import { log } from '../log.js';
+import type { MultimodalLLMMetrics } from '../metrics/base.js';
 import { BasicTranscriptionForwarder } from '../transcription.js';
 import { findMicroTrackId } from '../utils.js';
 import { AgentPlayout, type PlayoutHandle } from './agent_playout.js';
@@ -60,7 +61,7 @@ export class MultimodalAgent extends EventEmitter {
   room: Room | null = null;
   linkedParticipant: RemoteParticipant | null = null;
   subscribedTrack: RemoteAudioTrack | null = null;
-  readMicroTask: { promise: Promise<void>; cancel: () => void } | null = null;
+  readMicroTask: Promise<void> | null = null;
 
   constructor({
     model,
@@ -284,6 +285,7 @@ export class MultimodalAgent extends EventEmitter {
       });
 
       this.#session.on('input_speech_started', (ev: any) => {
+        this.emit('user_started_speaking');
         if (this.#playingHandle && !this.#playingHandle.done) {
           this.#playingHandle.interrupt();
 
@@ -326,6 +328,10 @@ export class MultimodalAgent extends EventEmitter {
         this.#updateState();
       });
 
+      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {
+        this.emit('metrics_collected', metrics);
+      });
+
       resolve(this.#session);
     });
   }
@@ -404,22 +410,11 @@ export class MultimodalAgent extends EventEmitter {
     };
     this.subscribedTrack = track;
 
-    if (this.readMicroTask) {
-      this.readMicroTask.cancel();
-    }
-
-    let cancel: () => void;
-    this.readMicroTask = {
-      promise: new Promise<void>((resolve, reject) => {
-        cancel = () => {
-          reject(new Error('Task cancelled'));
-        };
-        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
-          .then(resolve)
-          .catch(reject);
-      }),
-      cancel: () => cancel(),
-    };
+    this.readMicroTask = new Promise<void>((resolve, reject) => {
+      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+        .then(resolve)
+        .catch(reject);
+    });
   }
 
   #getLocalTrackSid(): string | null {

package/src/pipeline/index.ts

@@ -7,9 +7,9 @@ export {
   type BeforeTTSCallback,
   type BeforeLLMCallback,
   type VPACallbacks,
-  type AgentCallContext,
   type AgentTranscriptionOptions,
   type VPAOptions,
   VPAEvent,
   VoicePipelineAgent,
+  AgentCallContext,
 } from './pipeline_agent.js';