@livekit/agents 0.5.2 → 0.6.1

package/dist/metrics/base.d.ts

@@ -0,0 +1,96 @@
+export interface LLMMetrics {
+    requestId: string;
+    timestamp: number;
+    ttft: number;
+    duration: number;
+    label: string;
+    cancelled: boolean;
+    completionTokens: number;
+    promptTokens: number;
+    totalTokens: number;
+    tokensPerSecond: number;
+    error?: Error;
+}
+export interface STTMetrics {
+    requestId: string;
+    timestamp: number;
+    duration: number;
+    label: string;
+    audioDuration: number;
+    streamed: boolean;
+    error?: Error;
+}
+export interface TTSMetrics {
+    requestId: string;
+    timestamp: number;
+    ttfb: number;
+    duration: number;
+    label: string;
+    audioDuration: number;
+    cancelled: boolean;
+    charactersCount: number;
+    streamed: boolean;
+    error?: Error;
+}
+export interface VADMetrics {
+    timestamp: number;
+    idleTime: number;
+    inferenceDurationTotal: number;
+    inferenceCount: number;
+    label: string;
+}
+export interface PipelineEOUMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+    /** Timestamp of when the event was recorded */
+    timestamp: number;
+    /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+    endOfUtteranceDelay: number;
+    /**
+     * Time taken to obtain the transcript after the end of the user's speech.
+     *
+     * @remarks
+     * May be 0 if the transcript was already available.
+     */
+    transcriptionDelay: number;
+}
+export interface PipelineLLMMetrics extends LLMMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+}
+export interface PipelineTTSMetrics extends TTSMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+}
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+export declare class MultimodalLLMError extends Error {
+    type?: string;
+    reason?: string;
+    code?: string;
+    constructor({ type, reason, code, message, }?: {
+        type?: string;
+        reason?: string;
+        code?: string;
+        message?: string;
+    }, options?: ErrorOptions);
+}
+export interface MultimodalLLMMetrics extends LLMMetrics {
+    inputTokenDetails: {
+        cachedTokens: number;
+        textTokens: number;
+        audioTokens: number;
+    };
+    outputTokenDetails: {
+        textTokens: number;
+        audioTokens: number;
+    };
+}
+export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | PipelineSTTMetrics | PipelineEOUMetrics | PipelineLLMMetrics | PipelineTTSMetrics | PipelineVADMetrics | MultimodalLLMMetrics;
+//# sourceMappingURL=base.d.ts.map

package/dist/metrics/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/metrics/base.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,gGAAgG;IAChG,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,kBAAkB,GAAG,UAAU,CAAC;AAC5C,MAAM,MAAM,kBAAkB,GAAG,UAAU,CAAC;AAE5C,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;gBAEZ,EACE,IAAI,EACJ,MAAM,EACN,IAAI,EACJ,OAAO,GACR,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,EAC3E,OAAO,CAAC,EAAE,YAAY;CAOzB;AAED,MAAM,WAAW,oBAAqB,SAAQ,UAAU;IACtD,iBAAiB,EAAE;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,kBAAkB,EAAE;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,UAAU,GACV,UAAU,GACV,UAAU,GACV,kBAAkB,GAClB,kBAAkB,GAClB,kBAAkB,GAClB,kBAAkB,GAClB,kBAAkB,GAClB,oBAAoB,CAAC"}

package/dist/metrics/base.js

@@ -0,0 +1,20 @@
+class MultimodalLLMError extends Error {
+  type;
+  reason;
+  code;
+  constructor({
+    type,
+    reason,
+    code,
+    message
+  } = {}, options) {
+    super(message, options);
+    this.type = type;
+    this.reason = reason;
+    this.code = code;
+  }
+}
+export {
+  MultimodalLLMError
+};
+//# sourceMappingURL=base.js.map

package/dist/metrics/base.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/base.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport interface LLMMetrics {\n  requestId: string;\n  timestamp: number;\n  ttft: number;\n  duration: number;\n  label: string;\n  cancelled: boolean;\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n  tokensPerSecond: number;\n  error?: Error;\n}\n\nexport interface STTMetrics {\n  requestId: string;\n  timestamp: number;\n  duration: number;\n  label: string;\n  audioDuration: number;\n  streamed: boolean;\n  error?: Error;\n}\n\nexport interface TTSMetrics {\n  requestId: string;\n  timestamp: number;\n  ttfb: number;\n  duration: number;\n  label: string;\n  audioDuration: number;\n  cancelled: boolean;\n  charactersCount: number;\n  streamed: boolean;\n  error?: Error;\n}\n\nexport interface VADMetrics {\n  timestamp: number;\n  idleTime: number;\n  inferenceDurationTotal: number;\n  inferenceCount: number;\n  label: string;\n}\n\nexport interface PipelineEOUMetrics {\n  /**\n   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics\n   */\n  sequenceId: string;\n  /** Timestamp of when the event was recorded */\n  timestamp: number;\n  /** Amount of time between the end of speech from VAD and the decision to end the user's turn */\n  endOfUtteranceDelay: number;\n  /**\n   * Time taken to obtain the transcript after the end of the user's speech.\n   *\n   * @remarks\n   * May be 0 if the transcript was already available.\n   */\n  transcriptionDelay: number;\n}\n\nexport interface PipelineLLMMetrics extends LLMMetrics {\n  /**\n   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics\n   */\n  sequenceId: string;\n}\n\nexport interface PipelineTTSMetrics extends TTSMetrics {\n  /**\n   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics\n   */\n  sequenceId: string;\n}\n\nexport type PipelineSTTMetrics = STTMetrics;\nexport type PipelineVADMetrics = VADMetrics;\n\nexport class MultimodalLLMError extends Error {\n  type?: string;\n  reason?: string;\n  code?: string;\n  constructor(\n    {\n      type,\n      reason,\n      code,\n      message,\n    }: { type?: string; reason?: string; code?: string; message?: string } = {},\n    options?: ErrorOptions,\n  ) {\n    super(message, options);\n    this.type = type;\n    this.reason = reason;\n    this.code = code;\n  }\n}\n\nexport interface MultimodalLLMMetrics extends LLMMetrics {\n  inputTokenDetails: {\n    cachedTokens: number;\n    textTokens: number;\n    audioTokens: number;\n  };\n  outputTokenDetails: {\n    textTokens: number;\n    audioTokens: number;\n  };\n}\n\nexport type AgentMetrics =\n  | STTMetrics\n  | LLMMetrics\n  | TTSMetrics\n  | VADMetrics\n  | PipelineSTTMetrics\n  | PipelineEOUMetrics\n  | PipelineLLMMetrics\n  | PipelineTTSMetrics\n  | PipelineVADMetrics\n  | MultimodalLLMMetrics;\n"],"mappings":"AAoFO,MAAM,2BAA2B,MAAM;AAAA,EAC5C;AAAA,EACA;AAAA,EACA;AAAA,EACA,YACE;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAAyE,CAAC,GAC1E,SACA;AACA,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,SAAS;AACd,SAAK,OAAO;AAAA,EACd;AACF;","names":[]}

package/dist/metrics/index.cjs

@@ -0,0 +1,35 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var metrics_exports = {};
+__export(metrics_exports, {
+  MultimodalLLMError: () => import_base.MultimodalLLMError,
+  UsageCollector: () => import_usage_collector.UsageCollector,
+  logMetrics: () => import_utils.logMetrics
+});
+module.exports = __toCommonJS(metrics_exports);
+var import_base = require("./base.cjs");
+var import_usage_collector = require("./usage_collector.cjs");
+var import_utils = require("./utils.cjs");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MultimodalLLMError,
+  UsageCollector,
+  logMetrics
+});
+//# sourceMappingURL=index.cjs.map

package/dist/metrics/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type {\n  AgentMetrics,\n  STTMetrics,\n  LLMMetrics,\n  TTSMetrics,\n  VADMetrics,\n  PipelineSTTMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  PipelineVADMetrics,\n  MultimodalLLMMetrics,\n} from './base.js';\nexport { MultimodalLLMError } from './base.js';\nexport { type UsageSummary, UsageCollector } from './usage_collector.js';\nexport { logMetrics } from './utils.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBA,kBAAmC;AACnC,6BAAkD;AAClD,mBAA2B;","names":[]}

package/dist/metrics/index.d.ts

@@ -0,0 +1,5 @@
+export type { AgentMetrics, STTMetrics, LLMMetrics, TTSMetrics, VADMetrics, PipelineSTTMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, PipelineVADMetrics, MultimodalLLMMetrics, } from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAIA,YAAY,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,KAAK,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC"}

package/dist/metrics/index.js

@@ -0,0 +1,9 @@
+import { MultimodalLLMError } from "./base.js";
+import { UsageCollector } from "./usage_collector.js";
+import { logMetrics } from "./utils.js";
+export {
+  MultimodalLLMError,
+  UsageCollector,
+  logMetrics
+};
+//# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type {\n  AgentMetrics,\n  STTMetrics,\n  LLMMetrics,\n  TTSMetrics,\n  VADMetrics,\n  PipelineSTTMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  PipelineVADMetrics,\n  MultimodalLLMMetrics,\n} from './base.js';\nexport { MultimodalLLMError } from './base.js';\nexport { type UsageSummary, UsageCollector } from './usage_collector.js';\nexport { logMetrics } from './utils.js';\n"],"mappings":"AAiBA,SAAS,0BAA0B;AACnC,SAA4B,sBAAsB;AAClD,SAAS,kBAAkB;","names":[]}

package/dist/metrics/usage_collector.cjs

@@ -0,0 +1,53 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var usage_collector_exports = {};
+__export(usage_collector_exports, {
+  UsageCollector: () => UsageCollector
+});
+module.exports = __toCommonJS(usage_collector_exports);
+var import_utils = require("./utils.cjs");
+class UsageCollector {
+  #summary;
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0
+    };
+  }
+  collect(metrics) {
+    if ((0, import_utils.isLLMMetrics)(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if ((0, import_utils.isTTSMetrics)(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if ((0, import_utils.isSTTMetrics)(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+  get summary() {
+    return { ...this.#summary };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  UsageCollector
+});
+//# sourceMappingURL=usage_collector.cjs.map

package/dist/metrics/usage_collector.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/usage_collector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AgentMetrics } from './base.js';\nimport { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';\n\nexport interface UsageSummary {\n  llmPromptTokens: number;\n  llmCompletionTokens: number;\n  ttsCharactersCount: number;\n  sttAudioDuration: number;\n}\n\nexport class UsageCollector {\n  #summary: UsageSummary;\n\n  constructor() {\n    this.#summary = {\n      llmPromptTokens: 0,\n      llmCompletionTokens: 0,\n      ttsCharactersCount: 0,\n      sttAudioDuration: 0,\n    };\n  }\n\n  collect(metrics: AgentMetrics) {\n    if (isLLMMetrics(metrics)) {\n      this.#summary.llmPromptTokens += metrics.promptTokens;\n      this.#summary.llmCompletionTokens += metrics.completionTokens;\n    } else if (isTTSMetrics(metrics)) {\n      this.#summary.ttsCharactersCount += metrics.charactersCount;\n    } else if (isSTTMetrics(metrics)) {\n      this.#summary.sttAudioDuration += metrics.audioDuration;\n    }\n  }\n\n  get summary(): UsageSummary {\n    return { ...this.#summary };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAyD;AASlD,MAAM,eAAe;AAAA,EAC1B;AAAA,EAEA,cAAc;AACZ,SAAK,WAAW;AAAA,MACd,iBAAiB;AAAA,MACjB,qBAAqB;AAAA,MACrB,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,QAAQ,SAAuB;AAC7B,YAAI,2BAAa,OAAO,GAAG;AACzB,WAAK,SAAS,mBAAmB,QAAQ;AACzC,WAAK,SAAS,uBAAuB,QAAQ;AAAA,IAC/C,eAAW,2BAAa,OAAO,GAAG;AAChC,WAAK,SAAS,sBAAsB,QAAQ;AAAA,IAC9C,eAAW,2BAAa,OAAO,GAAG;AAChC,WAAK,SAAS,oBAAoB,QAAQ;AAAA,IAC5C;AAAA,EACF;AAAA,EAEA,IAAI,UAAwB;AAC1B,WAAO,EAAE,GAAG,KAAK,SAAS;AAAA,EAC5B;AACF;","names":[]}

package/dist/metrics/usage_collector.d.ts

@@ -0,0 +1,14 @@
+import type { AgentMetrics } from './base.js';
+export interface UsageSummary {
+    llmPromptTokens: number;
+    llmCompletionTokens: number;
+    ttsCharactersCount: number;
+    sttAudioDuration: number;
+}
+export declare class UsageCollector {
+    #private;
+    constructor();
+    collect(metrics: AgentMetrics): void;
+    get summary(): UsageSummary;
+}
+//# sourceMappingURL=usage_collector.d.ts.map

package/dist/metrics/usage_collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage_collector.d.ts","sourceRoot":"","sources":["../../src/metrics/usage_collector.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAG9C,MAAM,WAAW,YAAY;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,cAAc;;;IAYzB,OAAO,CAAC,OAAO,EAAE,YAAY;IAW7B,IAAI,OAAO,IAAI,YAAY,CAE1B;CACF"}

package/dist/metrics/usage_collector.js

@@ -0,0 +1,29 @@
+import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from "./utils.js";
+class UsageCollector {
+  #summary;
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0
+    };
+  }
+  collect(metrics) {
+    if (isLLMMetrics(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (isTTSMetrics(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (isSTTMetrics(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+  get summary() {
+    return { ...this.#summary };
+  }
+}
+export {
+  UsageCollector
+};
+//# sourceMappingURL=usage_collector.js.map

package/dist/metrics/usage_collector.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/usage_collector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AgentMetrics } from './base.js';\nimport { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';\n\nexport interface UsageSummary {\n  llmPromptTokens: number;\n  llmCompletionTokens: number;\n  ttsCharactersCount: number;\n  sttAudioDuration: number;\n}\n\nexport class UsageCollector {\n  #summary: UsageSummary;\n\n  constructor() {\n    this.#summary = {\n      llmPromptTokens: 0,\n      llmCompletionTokens: 0,\n      ttsCharactersCount: 0,\n      sttAudioDuration: 0,\n    };\n  }\n\n  collect(metrics: AgentMetrics) {\n    if (isLLMMetrics(metrics)) {\n      this.#summary.llmPromptTokens += metrics.promptTokens;\n      this.#summary.llmCompletionTokens += metrics.completionTokens;\n    } else if (isTTSMetrics(metrics)) {\n      this.#summary.ttsCharactersCount += metrics.charactersCount;\n    } else if (isSTTMetrics(metrics)) {\n      this.#summary.sttAudioDuration += metrics.audioDuration;\n    }\n  }\n\n  get summary(): UsageSummary {\n    return { ...this.#summary };\n  }\n}\n"],"mappings":"AAIA,SAAS,cAAc,cAAc,oBAAoB;AASlD,MAAM,eAAe;AAAA,EAC1B;AAAA,EAEA,cAAc;AACZ,SAAK,WAAW;AAAA,MACd,iBAAiB;AAAA,MACjB,qBAAqB;AAAA,MACrB,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,QAAQ,SAAuB;AAC7B,QAAI,aAAa,OAAO,GAAG;AACzB,WAAK,SAAS,mBAAmB,QAAQ;AACzC,WAAK,SAAS,uBAAuB,QAAQ;AAAA,IAC/C,WAAW,aAAa,OAAO,GAAG;AAChC,WAAK,SAAS,sBAAsB,QAAQ;AAAA,IAC9C,WAAW,aAAa,OAAO,GAAG;AAChC,WAAK,SAAS,oBAAoB,QAAQ;AAAA,IAC5C;AAAA,EACF;AAAA,EAEA,IAAI,UAAwB;AAC1B,WAAO,EAAE,GAAG,KAAK,SAAS;AAAA,EAC5B;AACF;","names":[]}

package/dist/metrics/utils.cjs

@@ -0,0 +1,104 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var utils_exports = {};
+__export(utils_exports, {
+  isLLMMetrics: () => isLLMMetrics,
+  isPipelineEOUMetrics: () => isPipelineEOUMetrics,
+  isPipelineLLMMetrics: () => isPipelineLLMMetrics,
+  isPipelineTTSMetrics: () => isPipelineTTSMetrics,
+  isSTTMetrics: () => isSTTMetrics,
+  isTTSMetrics: () => isTTSMetrics,
+  isVADMetrics: () => isVADMetrics,
+  logMetrics: () => logMetrics
+});
+module.exports = __toCommonJS(utils_exports);
+var import_log = require("../log.cjs");
+const logMetrics = (metrics) => {
+  const logger = (0, import_log.log)();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("Pipeline LLM metrics");
+  } else if (isLLMMetrics(metrics)) {
+    logger.child({
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("LLM metrics");
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("Pipeline TTS metrics");
+  } else if (isTTSMetrics(metrics)) {
+    logger.child({
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("TTS metrics");
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+      transcriptionDelay: metrics.transcriptionDelay
+    }).info("Pipeline EOU metrics");
+  } else if (isSTTMetrics(metrics)) {
+    logger.child({
+      audioDuration: metrics.audioDuration
+    }).info("STT metrics");
+  }
+};
+const isLLMMetrics = (metrics) => {
+  return !!metrics.ttft;
+};
+const isPipelineLLMMetrics = (metrics) => {
+  return isLLMMetrics(metrics) && !!metrics.sequenceId;
+};
+const isVADMetrics = (metrics) => {
+  return !!metrics.inferenceCount;
+};
+const isPipelineEOUMetrics = (metrics) => {
+  return !!metrics.endOfUtteranceDelay;
+};
+const isTTSMetrics = (metrics) => {
+  return !!metrics.ttfb;
+};
+const isPipelineTTSMetrics = (metrics) => {
+  return isTTSMetrics(metrics) && !!metrics.sequenceId;
+};
+const isSTTMetrics = (metrics) => {
+  return !(isLLMMetrics(metrics) || isVADMetrics(metrics) || isPipelineEOUMetrics(metrics) || isTTSMetrics(metrics));
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  isLLMMetrics,
+  isPipelineEOUMetrics,
+  isPipelineLLMMetrics,
+  isPipelineTTSMetrics,
+  isSTTMetrics,
+  isTTSMetrics,
+  isVADMetrics,
+  logMetrics
+});
+//# sourceMappingURL=utils.cjs.map

package/dist/metrics/utils.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { log } from '../log.js';\nimport type {\n  AgentMetrics,\n  LLMMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  STTMetrics,\n  TTSMetrics,\n  VADMetrics,\n} from './base.js';\n\nexport const logMetrics = (metrics: AgentMetrics) => {\n  const logger = log();\n  if (isPipelineLLMMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('Pipeline LLM metrics');\n  } else if (isLLMMetrics(metrics)) {\n    logger\n      .child({\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('LLM metrics');\n  } else if (isPipelineTTSMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('Pipeline TTS metrics');\n  } else if (isTTSMetrics(metrics)) {\n    logger\n      .child({\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('TTS metrics');\n  } else if (isPipelineEOUMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        endOfUtteranceDelay: metrics.endOfUtteranceDelay,\n        transcriptionDelay: metrics.transcriptionDelay,\n      })\n      .info('Pipeline EOU metrics');\n  } else if (isSTTMetrics(metrics)) {\n    logger\n      .child({\n        audioDuration: metrics.audioDuration,\n      })\n      .info('STT metrics');\n  }\n};\n\nexport const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {\n  return !!(metrics as LLMMetrics).ttft;\n};\n\nexport const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {\n  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;\n};\n\nexport const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {\n  return !!(metrics as VADMetrics).inferenceCount;\n};\n\nexport const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {\n  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;\n};\n\nexport const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {\n  return !!(metrics as TTSMetrics).ttfb;\n};\n\nexport const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {\n  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;\n};\n\nexport const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {\n  return !(\n    isLLMMetrics(metrics) ||\n    isVADMetrics(metrics) ||\n    isPipelineEOUMetrics(metrics) ||\n    isTTSMetrics(metrics)\n  );\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAoB;AAYb,MAAM,aAAa,CAAC,YAA0B;AACnD,QAAM,aAAS,gBAAI;AACnB,MAAI,qBAAqB,OAAO,GAAG;AACjC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,qBAAqB,QAAQ;AAAA,MAC7B,oBAAoB,QAAQ;AAAA,IAC9B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB;AACF;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,CAAC,CAAE,QAA+B;AAC3C;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,EACL,aAAa,OAAO,KACpB,aAAa,OAAO,KACpB,qBAAqB,OAAO,KAC5B,aAAa,OAAO;AAExB;","names":[]}

package/dist/metrics/utils.d.ts

@@ -0,0 +1,10 @@
+import type { AgentMetrics, LLMMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, STTMetrics, TTSMetrics, VADMetrics } from './base.js';
+export declare const logMetrics: (metrics: AgentMetrics) => void;
+export declare const isLLMMetrics: (metrics: AgentMetrics) => metrics is LLMMetrics;
+export declare const isPipelineLLMMetrics: (metrics: AgentMetrics) => metrics is PipelineLLMMetrics;
+export declare const isVADMetrics: (metrics: AgentMetrics) => metrics is VADMetrics;
+export declare const isPipelineEOUMetrics: (metrics: AgentMetrics) => metrics is PipelineEOUMetrics;
+export declare const isTTSMetrics: (metrics: AgentMetrics) => metrics is TTSMetrics;
+export declare const isPipelineTTSMetrics: (metrics: AgentMetrics) => metrics is PipelineTTSMetrics;
+export declare const isSTTMetrics: (metrics: AgentMetrics) => metrics is STTMetrics;
+//# sourceMappingURL=utils.d.ts.map

package/dist/metrics/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/metrics/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,EACV,UAAU,EACV,UAAU,EACX,MAAM,WAAW,CAAC;AAEnB,eAAO,MAAM,UAAU,YAAa,YAAY,SAmD/C,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAEjD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YAAa,YAAY,kCAEzD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAEjD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YAAa,YAAY,kCAEzD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAEjD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YAAa,YAAY,kCAEzD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAOjD,CAAC"}

package/dist/metrics/utils.js

@@ -0,0 +1,73 @@
+import { log } from "../log.js";
+const logMetrics = (metrics) => {
+  const logger = log();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("Pipeline LLM metrics");
+  } else if (isLLMMetrics(metrics)) {
+    logger.child({
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("LLM metrics");
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("Pipeline TTS metrics");
+  } else if (isTTSMetrics(metrics)) {
+    logger.child({
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("TTS metrics");
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+      transcriptionDelay: metrics.transcriptionDelay
+    }).info("Pipeline EOU metrics");
+  } else if (isSTTMetrics(metrics)) {
+    logger.child({
+      audioDuration: metrics.audioDuration
+    }).info("STT metrics");
+  }
+};
+const isLLMMetrics = (metrics) => {
+  return !!metrics.ttft;
+};
+const isPipelineLLMMetrics = (metrics) => {
+  return isLLMMetrics(metrics) && !!metrics.sequenceId;
+};
+const isVADMetrics = (metrics) => {
+  return !!metrics.inferenceCount;
+};
+const isPipelineEOUMetrics = (metrics) => {
+  return !!metrics.endOfUtteranceDelay;
+};
+const isTTSMetrics = (metrics) => {
+  return !!metrics.ttfb;
+};
+const isPipelineTTSMetrics = (metrics) => {
+  return isTTSMetrics(metrics) && !!metrics.sequenceId;
+};
+const isSTTMetrics = (metrics) => {
+  return !(isLLMMetrics(metrics) || isVADMetrics(metrics) || isPipelineEOUMetrics(metrics) || isTTSMetrics(metrics));
+};
+export {
+  isLLMMetrics,
+  isPipelineEOUMetrics,
+  isPipelineLLMMetrics,
+  isPipelineTTSMetrics,
+  isSTTMetrics,
+  isTTSMetrics,
+  isVADMetrics,
+  logMetrics
+};
+//# sourceMappingURL=utils.js.map

package/dist/metrics/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { log } from '../log.js';\nimport type {\n  AgentMetrics,\n  LLMMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  STTMetrics,\n  TTSMetrics,\n  VADMetrics,\n} from './base.js';\n\nexport const logMetrics = (metrics: AgentMetrics) => {\n  const logger = log();\n  if (isPipelineLLMMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('Pipeline LLM metrics');\n  } else if (isLLMMetrics(metrics)) {\n    logger\n      .child({\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('LLM metrics');\n  } else if (isPipelineTTSMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('Pipeline TTS metrics');\n  } else if (isTTSMetrics(metrics)) {\n    logger\n      .child({\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('TTS metrics');\n  } else if (isPipelineEOUMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        endOfUtteranceDelay: metrics.endOfUtteranceDelay,\n        transcriptionDelay: metrics.transcriptionDelay,\n      })\n      .info('Pipeline EOU metrics');\n  } else if (isSTTMetrics(metrics)) {\n    logger\n      .child({\n        audioDuration: metrics.audioDuration,\n      })\n      .info('STT metrics');\n  }\n};\n\nexport const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {\n  return !!(metrics as LLMMetrics).ttft;\n};\n\nexport const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {\n  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;\n};\n\nexport const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {\n  return !!(metrics as VADMetrics).inferenceCount;\n};\n\nexport const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {\n  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;\n};\n\nexport const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {\n  return !!(metrics as TTSMetrics).ttfb;\n};\n\nexport const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {\n  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;\n};\n\nexport const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {\n  return !(\n    isLLMMetrics(metrics) ||\n    isVADMetrics(metrics) ||\n    isPipelineEOUMetrics(metrics) ||\n    isTTSMetrics(metrics)\n  );\n};\n"],"mappings":"AAGA,SAAS,WAAW;AAYb,MAAM,aAAa,CAAC,YAA0B;AACnD,QAAM,SAAS,IAAI;AACnB,MAAI,qBAAqB,OAAO,GAAG;AACjC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,qBAAqB,QAAQ;AAAA,MAC7B,oBAAoB,QAAQ;AAAA,IAC9B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB;AACF;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,CAAC,CAAE,QAA+B;AAC3C;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,EACL,aAAa,OAAO,KACpB,aAAa,OAAO,KACpB,qBAAqB,OAAO,KAC5B,aAAa,OAAO;AAExB;","names":[]}

package/dist/multimodal/multimodal_agent.cjs

@@ -53,15 +53,19 @@ class MultimodalAgent extends import_node_events.EventEmitter {
   linkedParticipant = null;
   subscribedTrack = null;
   readMicroTask = null;
+  #textResponseRetries = 0;
+  #maxTextResponseRetries;
   constructor({
     model,
     chatCtx,
-    fncCtx
+    fncCtx,
+    maxTextResponseRetries = 5
   }) {
     super();
     this.model = model;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.#maxTextResponseRetries = maxTextResponseRetries;
   }
   #participant = null;
   #agentPublication = null;
@@ -190,6 +194,7 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       this.#started = true;
       this.#session.on("response_content_added", (message) => {
         var _a2;
+        if (message.contentType === "text") return;
         const trFwd = new import_transcription.BasicTranscriptionForwarder(
           this.room,
           this.room.localParticipant.identity,
@@ -205,6 +210,26 @@ class MultimodalAgent extends import_node_events.EventEmitter {
         );
         this.#playingHandle = handle;
       });
+      this.#session.on("response_content_done", (message) => {
+        if (message.contentType === "text") {
+          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {
+            throw new Error(
+              `The OpenAI Realtime API returned a text response after ${this.#maxTextResponseRetries} retries. Please try to reduce the number of text system or assistant messages in the chat context.`
+            );
+          }
+          this.#textResponseRetries++;
+          this.#logger.child({
+            itemId: message.itemId,
+            text: message.text,
+            retries: this.#textResponseRetries
+          }).warn(
+            "The OpenAI Realtime API returned a text response instead of audio. Attempting to recover to audio mode..."
+          );
+          this.#session.recoverFromTextResponse(message.itemId);
+        } else {
+          this.#textResponseRetries = 0;
+        }
+      });
       this.#session.on("input_speech_committed", (ev) => {
         var _a2, _b;
         const participantIdentity = (_a2 = this.linkedParticipant) == null ? void 0 : _a2.identity;
@@ -234,6 +259,7 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       });
       this.#session.on("input_speech_started", (ev) => {
         var _a2, _b;
+        this.emit("user_started_speaking");
         if (this.#playingHandle && !this.#playingHandle.done) {
           this.#playingHandle.interrupt();
           this.#session.conversation.item.truncate(
@@ -264,6 +290,9 @@ class MultimodalAgent extends import_node_events.EventEmitter {
         this.#pendingFunctionCalls.delete(ev.callId);
         this.#updateState();
       });
+      this.#session.on("metrics_collected", (metrics) => {
+        this.emit("metrics_collected", metrics);
+      });
       resolve(this.#session);
     });
   }
@@ -325,24 +354,13 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       }
     };
     this.subscribedTrack = track;
-    if (this.readMicroTask) {
-      this.readMicroTask.cancel();
-    }
-    let cancel;
-    this.readMicroTask = {
-      promise: new Promise((resolve, reject) => {
-        cancel = () => {
-          reject(new Error("Task cancelled"));
-        };
-        readAudioStreamTask(new import_rtc_node.AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
-      }),
-      cancel: () => cancel()
-    };
+    this.readMicroTask = new Promise((resolve, reject) => {
+      readAudioStreamTask(new import_rtc_node.AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
+    });
   }
   #getLocalTrackSid() {
-    var _a;
     if (!this.#localTrackSid && this.room && this.room.localParticipant) {
-      this.#localTrackSid = (0, import_utils.findMicroTrackId)(this.room, (_a = this.room.localParticipant) == null ? void 0 : _a.identity);
+      this.#localTrackSid = (0, import_utils.findMicroTrackId)(this.room, this.room.localParticipant.identity);
     }
     return this.#localTrackSid;
   }