@livekit/agents 0.5.1 → 0.6.0

package/dist/metrics/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type {\n  AgentMetrics,\n  STTMetrics,\n  LLMMetrics,\n  TTSMetrics,\n  VADMetrics,\n  PipelineSTTMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  PipelineVADMetrics,\n  MultimodalLLMMetrics,\n} from './base.js';\nexport { MultimodalLLMError } from './base.js';\nexport { type UsageSummary, UsageCollector } from './usage_collector.js';\nexport { logMetrics } from './utils.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBA,kBAAmC;AACnC,6BAAkD;AAClD,mBAA2B;","names":[]}

package/dist/metrics/index.d.ts

@@ -0,0 +1,5 @@
+export type { AgentMetrics, STTMetrics, LLMMetrics, TTSMetrics, VADMetrics, PipelineSTTMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, PipelineVADMetrics, MultimodalLLMMetrics, } from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAIA,YAAY,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,KAAK,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC"}

package/dist/metrics/index.js

@@ -0,0 +1,9 @@
+import { MultimodalLLMError } from "./base.js";
+import { UsageCollector } from "./usage_collector.js";
+import { logMetrics } from "./utils.js";
+export {
+  MultimodalLLMError,
+  UsageCollector,
+  logMetrics
+};
+//# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type {\n  AgentMetrics,\n  STTMetrics,\n  LLMMetrics,\n  TTSMetrics,\n  VADMetrics,\n  PipelineSTTMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  PipelineVADMetrics,\n  MultimodalLLMMetrics,\n} from './base.js';\nexport { MultimodalLLMError } from './base.js';\nexport { type UsageSummary, UsageCollector } from './usage_collector.js';\nexport { logMetrics } from './utils.js';\n"],"mappings":"AAiBA,SAAS,0BAA0B;AACnC,SAA4B,sBAAsB;AAClD,SAAS,kBAAkB;","names":[]}

package/dist/metrics/usage_collector.cjs

@@ -0,0 +1,53 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var usage_collector_exports = {};
+__export(usage_collector_exports, {
+  UsageCollector: () => UsageCollector
+});
+module.exports = __toCommonJS(usage_collector_exports);
+var import_utils = require("./utils.cjs");
+class UsageCollector {
+  #summary;
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0
+    };
+  }
+  collect(metrics) {
+    if ((0, import_utils.isLLMMetrics)(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if ((0, import_utils.isTTSMetrics)(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if ((0, import_utils.isSTTMetrics)(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+  get summary() {
+    return { ...this.#summary };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  UsageCollector
+});
+//# sourceMappingURL=usage_collector.cjs.map

package/dist/metrics/usage_collector.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/usage_collector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AgentMetrics } from './base.js';\nimport { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';\n\nexport interface UsageSummary {\n  llmPromptTokens: number;\n  llmCompletionTokens: number;\n  ttsCharactersCount: number;\n  sttAudioDuration: number;\n}\n\nexport class UsageCollector {\n  #summary: UsageSummary;\n\n  constructor() {\n    this.#summary = {\n      llmPromptTokens: 0,\n      llmCompletionTokens: 0,\n      ttsCharactersCount: 0,\n      sttAudioDuration: 0,\n    };\n  }\n\n  collect(metrics: AgentMetrics) {\n    if (isLLMMetrics(metrics)) {\n      this.#summary.llmPromptTokens += metrics.promptTokens;\n      this.#summary.llmCompletionTokens += metrics.completionTokens;\n    } else if (isTTSMetrics(metrics)) {\n      this.#summary.ttsCharactersCount += metrics.charactersCount;\n    } else if (isSTTMetrics(metrics)) {\n      this.#summary.sttAudioDuration += metrics.audioDuration;\n    }\n  }\n\n  get summary(): UsageSummary {\n    return { ...this.#summary };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAyD;AASlD,MAAM,eAAe;AAAA,EAC1B;AAAA,EAEA,cAAc;AACZ,SAAK,WAAW;AAAA,MACd,iBAAiB;AAAA,MACjB,qBAAqB;AAAA,MACrB,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,QAAQ,SAAuB;AAC7B,YAAI,2BAAa,OAAO,GAAG;AACzB,WAAK,SAAS,mBAAmB,QAAQ;AACzC,WAAK,SAAS,uBAAuB,QAAQ;AAAA,IAC/C,eAAW,2BAAa,OAAO,GAAG;AAChC,WAAK,SAAS,sBAAsB,QAAQ;AAAA,IAC9C,eAAW,2BAAa,OAAO,GAAG;AAChC,WAAK,SAAS,oBAAoB,QAAQ;AAAA,IAC5C;AAAA,EACF;AAAA,EAEA,IAAI,UAAwB;AAC1B,WAAO,EAAE,GAAG,KAAK,SAAS;AAAA,EAC5B;AACF;","names":[]}

package/dist/metrics/usage_collector.d.ts

@@ -0,0 +1,14 @@
+import type { AgentMetrics } from './base.js';
+export interface UsageSummary {
+    llmPromptTokens: number;
+    llmCompletionTokens: number;
+    ttsCharactersCount: number;
+    sttAudioDuration: number;
+}
+export declare class UsageCollector {
+    #private;
+    constructor();
+    collect(metrics: AgentMetrics): void;
+    get summary(): UsageSummary;
+}
+//# sourceMappingURL=usage_collector.d.ts.map

package/dist/metrics/usage_collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage_collector.d.ts","sourceRoot":"","sources":["../../src/metrics/usage_collector.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAG9C,MAAM,WAAW,YAAY;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,cAAc;;;IAYzB,OAAO,CAAC,OAAO,EAAE,YAAY;IAW7B,IAAI,OAAO,IAAI,YAAY,CAE1B;CACF"}

package/dist/metrics/usage_collector.js

@@ -0,0 +1,29 @@
+import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from "./utils.js";
+class UsageCollector {
+  #summary;
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0
+    };
+  }
+  collect(metrics) {
+    if (isLLMMetrics(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (isTTSMetrics(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (isSTTMetrics(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+  get summary() {
+    return { ...this.#summary };
+  }
+}
+export {
+  UsageCollector
+};
+//# sourceMappingURL=usage_collector.js.map

package/dist/metrics/usage_collector.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/usage_collector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AgentMetrics } from './base.js';\nimport { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';\n\nexport interface UsageSummary {\n  llmPromptTokens: number;\n  llmCompletionTokens: number;\n  ttsCharactersCount: number;\n  sttAudioDuration: number;\n}\n\nexport class UsageCollector {\n  #summary: UsageSummary;\n\n  constructor() {\n    this.#summary = {\n      llmPromptTokens: 0,\n      llmCompletionTokens: 0,\n      ttsCharactersCount: 0,\n      sttAudioDuration: 0,\n    };\n  }\n\n  collect(metrics: AgentMetrics) {\n    if (isLLMMetrics(metrics)) {\n      this.#summary.llmPromptTokens += metrics.promptTokens;\n      this.#summary.llmCompletionTokens += metrics.completionTokens;\n    } else if (isTTSMetrics(metrics)) {\n      this.#summary.ttsCharactersCount += metrics.charactersCount;\n    } else if (isSTTMetrics(metrics)) {\n      this.#summary.sttAudioDuration += metrics.audioDuration;\n    }\n  }\n\n  get summary(): UsageSummary {\n    return { ...this.#summary };\n  }\n}\n"],"mappings":"AAIA,SAAS,cAAc,cAAc,oBAAoB;AASlD,MAAM,eAAe;AAAA,EAC1B;AAAA,EAEA,cAAc;AACZ,SAAK,WAAW;AAAA,MACd,iBAAiB;AAAA,MACjB,qBAAqB;AAAA,MACrB,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,QAAQ,SAAuB;AAC7B,QAAI,aAAa,OAAO,GAAG;AACzB,WAAK,SAAS,mBAAmB,QAAQ;AACzC,WAAK,SAAS,uBAAuB,QAAQ;AAAA,IAC/C,WAAW,aAAa,OAAO,GAAG;AAChC,WAAK,SAAS,sBAAsB,QAAQ;AAAA,IAC9C,WAAW,aAAa,OAAO,GAAG;AAChC,WAAK,SAAS,oBAAoB,QAAQ;AAAA,IAC5C;AAAA,EACF;AAAA,EAEA,IAAI,UAAwB;AAC1B,WAAO,EAAE,GAAG,KAAK,SAAS;AAAA,EAC5B;AACF;","names":[]}

package/dist/metrics/utils.cjs

@@ -0,0 +1,104 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var utils_exports = {};
+__export(utils_exports, {
+  isLLMMetrics: () => isLLMMetrics,
+  isPipelineEOUMetrics: () => isPipelineEOUMetrics,
+  isPipelineLLMMetrics: () => isPipelineLLMMetrics,
+  isPipelineTTSMetrics: () => isPipelineTTSMetrics,
+  isSTTMetrics: () => isSTTMetrics,
+  isTTSMetrics: () => isTTSMetrics,
+  isVADMetrics: () => isVADMetrics,
+  logMetrics: () => logMetrics
+});
+module.exports = __toCommonJS(utils_exports);
+var import_log = require("../log.cjs");
+const logMetrics = (metrics) => {
+  const logger = (0, import_log.log)();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("Pipeline LLM metrics");
+  } else if (isLLMMetrics(metrics)) {
+    logger.child({
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("LLM metrics");
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("Pipeline TTS metrics");
+  } else if (isTTSMetrics(metrics)) {
+    logger.child({
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("TTS metrics");
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+      transcriptionDelay: metrics.transcriptionDelay
+    }).info("Pipeline EOU metrics");
+  } else if (isSTTMetrics(metrics)) {
+    logger.child({
+      audioDuration: metrics.audioDuration
+    }).info("STT metrics");
+  }
+};
+const isLLMMetrics = (metrics) => {
+  return !!metrics.ttft;
+};
+const isPipelineLLMMetrics = (metrics) => {
+  return isLLMMetrics(metrics) && !!metrics.sequenceId;
+};
+const isVADMetrics = (metrics) => {
+  return !!metrics.inferenceCount;
+};
+const isPipelineEOUMetrics = (metrics) => {
+  return !!metrics.endOfUtteranceDelay;
+};
+const isTTSMetrics = (metrics) => {
+  return !!metrics.ttfb;
+};
+const isPipelineTTSMetrics = (metrics) => {
+  return isTTSMetrics(metrics) && !!metrics.sequenceId;
+};
+const isSTTMetrics = (metrics) => {
+  return !(isLLMMetrics(metrics) || isVADMetrics(metrics) || isPipelineEOUMetrics(metrics) || isTTSMetrics(metrics));
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  isLLMMetrics,
+  isPipelineEOUMetrics,
+  isPipelineLLMMetrics,
+  isPipelineTTSMetrics,
+  isSTTMetrics,
+  isTTSMetrics,
+  isVADMetrics,
+  logMetrics
+});
+//# sourceMappingURL=utils.cjs.map

package/dist/metrics/utils.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { log } from '../log.js';\nimport type {\n  AgentMetrics,\n  LLMMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  STTMetrics,\n  TTSMetrics,\n  VADMetrics,\n} from './base.js';\n\nexport const logMetrics = (metrics: AgentMetrics) => {\n  const logger = log();\n  if (isPipelineLLMMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('Pipeline LLM metrics');\n  } else if (isLLMMetrics(metrics)) {\n    logger\n      .child({\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('LLM metrics');\n  } else if (isPipelineTTSMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('Pipeline TTS metrics');\n  } else if (isTTSMetrics(metrics)) {\n    logger\n      .child({\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('TTS metrics');\n  } else if (isPipelineEOUMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        endOfUtteranceDelay: metrics.endOfUtteranceDelay,\n        transcriptionDelay: metrics.transcriptionDelay,\n      })\n      .info('Pipeline EOU metrics');\n  } else if (isSTTMetrics(metrics)) {\n    logger\n      .child({\n        audioDuration: metrics.audioDuration,\n      })\n      .info('STT metrics');\n  }\n};\n\nexport const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {\n  return !!(metrics as LLMMetrics).ttft;\n};\n\nexport const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {\n  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;\n};\n\nexport const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {\n  return !!(metrics as VADMetrics).inferenceCount;\n};\n\nexport const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {\n  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;\n};\n\nexport const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {\n  return !!(metrics as TTSMetrics).ttfb;\n};\n\nexport const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {\n  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;\n};\n\nexport const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {\n  return !(\n    isLLMMetrics(metrics) ||\n    isVADMetrics(metrics) ||\n    isPipelineEOUMetrics(metrics) ||\n    isTTSMetrics(metrics)\n  );\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAoB;AAYb,MAAM,aAAa,CAAC,YAA0B;AACnD,QAAM,aAAS,gBAAI;AACnB,MAAI,qBAAqB,OAAO,GAAG;AACjC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,qBAAqB,QAAQ;AAAA,MAC7B,oBAAoB,QAAQ;AAAA,IAC9B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB;AACF;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,CAAC,CAAE,QAA+B;AAC3C;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,EACL,aAAa,OAAO,KACpB,aAAa,OAAO,KACpB,qBAAqB,OAAO,KAC5B,aAAa,OAAO;AAExB;","names":[]}

package/dist/metrics/utils.d.ts

@@ -0,0 +1,10 @@
+import type { AgentMetrics, LLMMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, STTMetrics, TTSMetrics, VADMetrics } from './base.js';
+export declare const logMetrics: (metrics: AgentMetrics) => void;
+export declare const isLLMMetrics: (metrics: AgentMetrics) => metrics is LLMMetrics;
+export declare const isPipelineLLMMetrics: (metrics: AgentMetrics) => metrics is PipelineLLMMetrics;
+export declare const isVADMetrics: (metrics: AgentMetrics) => metrics is VADMetrics;
+export declare const isPipelineEOUMetrics: (metrics: AgentMetrics) => metrics is PipelineEOUMetrics;
+export declare const isTTSMetrics: (metrics: AgentMetrics) => metrics is TTSMetrics;
+export declare const isPipelineTTSMetrics: (metrics: AgentMetrics) => metrics is PipelineTTSMetrics;
+export declare const isSTTMetrics: (metrics: AgentMetrics) => metrics is STTMetrics;
+//# sourceMappingURL=utils.d.ts.map

package/dist/metrics/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/metrics/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,EACV,UAAU,EACV,UAAU,EACX,MAAM,WAAW,CAAC;AAEnB,eAAO,MAAM,UAAU,YAAa,YAAY,SAmD/C,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAEjD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YAAa,YAAY,kCAEzD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAEjD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YAAa,YAAY,kCAEzD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAEjD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YAAa,YAAY,kCAEzD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,YAAY,0BAOjD,CAAC"}

package/dist/metrics/utils.js

@@ -0,0 +1,73 @@
+import { log } from "../log.js";
+const logMetrics = (metrics) => {
+  const logger = log();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("Pipeline LLM metrics");
+  } else if (isLLMMetrics(metrics)) {
+    logger.child({
+      ttft: metrics.ttft,
+      inputTokens: metrics.promptTokens,
+      outputTokens: metrics.completionTokens,
+      tokensPerSecond: metrics.tokensPerSecond
+    }).info("LLM metrics");
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("Pipeline TTS metrics");
+  } else if (isTTSMetrics(metrics)) {
+    logger.child({
+      ttfb: metrics.ttfb,
+      audioDuration: metrics.audioDuration
+    }).info("TTS metrics");
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger.child({
+      sequenceId: metrics.sequenceId,
+      endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+      transcriptionDelay: metrics.transcriptionDelay
+    }).info("Pipeline EOU metrics");
+  } else if (isSTTMetrics(metrics)) {
+    logger.child({
+      audioDuration: metrics.audioDuration
+    }).info("STT metrics");
+  }
+};
+const isLLMMetrics = (metrics) => {
+  return !!metrics.ttft;
+};
+const isPipelineLLMMetrics = (metrics) => {
+  return isLLMMetrics(metrics) && !!metrics.sequenceId;
+};
+const isVADMetrics = (metrics) => {
+  return !!metrics.inferenceCount;
+};
+const isPipelineEOUMetrics = (metrics) => {
+  return !!metrics.endOfUtteranceDelay;
+};
+const isTTSMetrics = (metrics) => {
+  return !!metrics.ttfb;
+};
+const isPipelineTTSMetrics = (metrics) => {
+  return isTTSMetrics(metrics) && !!metrics.sequenceId;
+};
+const isSTTMetrics = (metrics) => {
+  return !(isLLMMetrics(metrics) || isVADMetrics(metrics) || isPipelineEOUMetrics(metrics) || isTTSMetrics(metrics));
+};
+export {
+  isLLMMetrics,
+  isPipelineEOUMetrics,
+  isPipelineLLMMetrics,
+  isPipelineTTSMetrics,
+  isSTTMetrics,
+  isTTSMetrics,
+  isVADMetrics,
+  logMetrics
+};
+//# sourceMappingURL=utils.js.map

package/dist/metrics/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { log } from '../log.js';\nimport type {\n  AgentMetrics,\n  LLMMetrics,\n  PipelineEOUMetrics,\n  PipelineLLMMetrics,\n  PipelineTTSMetrics,\n  STTMetrics,\n  TTSMetrics,\n  VADMetrics,\n} from './base.js';\n\nexport const logMetrics = (metrics: AgentMetrics) => {\n  const logger = log();\n  if (isPipelineLLMMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('Pipeline LLM metrics');\n  } else if (isLLMMetrics(metrics)) {\n    logger\n      .child({\n        ttft: metrics.ttft,\n        inputTokens: metrics.promptTokens,\n        outputTokens: metrics.completionTokens,\n        tokensPerSecond: metrics.tokensPerSecond,\n      })\n      .info('LLM metrics');\n  } else if (isPipelineTTSMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('Pipeline TTS metrics');\n  } else if (isTTSMetrics(metrics)) {\n    logger\n      .child({\n        ttfb: metrics.ttfb,\n        audioDuration: metrics.audioDuration,\n      })\n      .info('TTS metrics');\n  } else if (isPipelineEOUMetrics(metrics)) {\n    logger\n      .child({\n        sequenceId: metrics.sequenceId,\n        endOfUtteranceDelay: metrics.endOfUtteranceDelay,\n        transcriptionDelay: metrics.transcriptionDelay,\n      })\n      .info('Pipeline EOU metrics');\n  } else if (isSTTMetrics(metrics)) {\n    logger\n      .child({\n        audioDuration: metrics.audioDuration,\n      })\n      .info('STT metrics');\n  }\n};\n\nexport const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {\n  return !!(metrics as LLMMetrics).ttft;\n};\n\nexport const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {\n  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;\n};\n\nexport const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {\n  return !!(metrics as VADMetrics).inferenceCount;\n};\n\nexport const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {\n  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;\n};\n\nexport const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {\n  return !!(metrics as TTSMetrics).ttfb;\n};\n\nexport const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {\n  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;\n};\n\nexport const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {\n  return !(\n    isLLMMetrics(metrics) ||\n    isVADMetrics(metrics) ||\n    isPipelineEOUMetrics(metrics) ||\n    isTTSMetrics(metrics)\n  );\n};\n"],"mappings":"AAGA,SAAS,WAAW;AAYb,MAAM,aAAa,CAAC,YAA0B;AACnD,QAAM,SAAS,IAAI;AACnB,MAAI,qBAAqB,OAAO,GAAG;AACjC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,aAAa,QAAQ;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,IAC3B,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB,WAAW,qBAAqB,OAAO,GAAG;AACxC,WACG,MAAM;AAAA,MACL,YAAY,QAAQ;AAAA,MACpB,qBAAqB,QAAQ;AAAA,MAC7B,oBAAoB,QAAQ;AAAA,IAC9B,CAAC,EACA,KAAK,sBAAsB;AAAA,EAChC,WAAW,aAAa,OAAO,GAAG;AAChC,WACG,MAAM;AAAA,MACL,eAAe,QAAQ;AAAA,IACzB,CAAC,EACA,KAAK,aAAa;AAAA,EACvB;AACF;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,CAAC,CAAE,QAA+B;AAC3C;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,CAAC,CAAE,QAAuB;AACnC;AAEO,MAAM,uBAAuB,CAAC,YAAyD;AAC5F,SAAO,aAAa,OAAO,KAAK,CAAC,CAAE,QAA+B;AACpE;AAEO,MAAM,eAAe,CAAC,YAAiD;AAC5E,SAAO,EACL,aAAa,OAAO,KACpB,aAAa,OAAO,KACpB,qBAAqB,OAAO,KAC5B,aAAa,OAAO;AAExB;","names":[]}

package/dist/multimodal/multimodal_agent.cjs

@@ -234,6 +234,7 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       });
       this.#session.on("input_speech_started", (ev) => {
         var _a2, _b;
+        this.emit("user_started_speaking");
         if (this.#playingHandle && !this.#playingHandle.done) {
           this.#playingHandle.interrupt();
           this.#session.conversation.item.truncate(
@@ -264,6 +265,9 @@ class MultimodalAgent extends import_node_events.EventEmitter {
         this.#pendingFunctionCalls.delete(ev.callId);
         this.#updateState();
       });
+      this.#session.on("metrics_collected", (metrics) => {
+        this.emit("metrics_collected", metrics);
+      });
       resolve(this.#session);
     });
   }
@@ -325,19 +329,9 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       }
     };
     this.subscribedTrack = track;
-    if (this.readMicroTask) {
-      this.readMicroTask.cancel();
-    }
-    let cancel;
-    this.readMicroTask = {
-      promise: new Promise((resolve, reject) => {
-        cancel = () => {
-          reject(new Error("Task cancelled"));
-        };
-        readAudioStreamTask(new import_rtc_node.AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
-      }),
-      cancel: () => cancel()
-    };
+    this.readMicroTask = new Promise((resolve, reject) => {
+      readAudioStreamTask(new import_rtc_node.AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
+    });
   }
   #getLocalTrackSid() {
     var _a;

package/dist/multimodal/multimodal_agent.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: { promise: Promise<void>; cancel: () => void } | null = null;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    if (this.readMicroTask) {\n      this.readMicroTask.cancel();\n    }\n\n    let cancel: () => void;\n    this.readMicroTask = {\n      promise: new Promise<void>((resolve, reject) => {\n        cancel = () => {\n          reject(new Error('Task cancelled'));\n        };\n        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n          .then(resolve)\n          .catch(reject);\n      }),\n      cancel: () => cancel(),\n    };\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAOO;AACP,yBAA6B;AAC7B,mBAAgC;AAChC,UAAqB;AACrB,iBAAoB;AACpB,2BAA4C;AAC5C,mBAAiC;AACjC,2BAAiD;AAM1C,MAAe,wBAAwB,gCAAa;AAM3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,gCAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAuE;AAAA,EAEvE,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,cAAU,gBAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAxIlD;AAyIM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,0BAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAQ;AAAA,MAC5C,CAAC;AACD,WAAK;AAAA,QACH,0BAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,4BAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,0BAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,4BAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oCAAoB;AACxC,cAAQ,SAAS,4BAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAQ;AAAA,QAC5C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAQ;AAC1C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AA3OnE,YAAAC;AA6OQ,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AA/P9D,YAAAA,KAAA;AAiQQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AA3Q5E,YAAAA,KAAA;AA6QQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AA7R5D,YAAAA,KAAA;AA8RQ,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AA/XJ;AAgYI,QACE,YAAY,WAAW,4BAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,QAAI;AACJ,SAAK,gBAAgB;AAAA,MACnB,SAAS,IAAI,QAAc,CAAC,SAAS,WAAW;AAC9C,iBAAS,MAAM;AACb,iBAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,QACpC;AACA,4BAAoB,IAAI,4BAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,MACjB,CAAC;AAAA,MACD,QAAQ,MAAM,OAAO;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,oBAAmC;AAxarC;AAyaI,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,qBAAiB,+BAAiB,KAAK,OAAM,UAAK,KAAK,qBAAV,mBAA4B,QAAQ;AAAA,IACxF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AArbV;AAsbI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AA3d/B;AA4dI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAW,qBAAqB;AAChF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}
+{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { MultimodalLLMMetrics } from '../metrics/base.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: Promise<void> | null = null;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        this.emit('user_started_speaking');\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {\n        this.emit('metrics_collected', metrics);\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    this.readMicroTask = new Promise<void>((resolve, reject) => {\n      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n        .then(resolve)\n        .catch(reject);\n    });\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAOO;AACP,yBAA6B;AAC7B,mBAAgC;AAChC,UAAqB;AACrB,iBAAoB;AAEpB,2BAA4C;AAC5C,mBAAiC;AACjC,2BAAiD;AAM1C,MAAe,wBAAwB,gCAAa;AAM3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,gCAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAsC;AAAA,EAEtC,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,cAAU,gBAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAzIlD;AA0IM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,0BAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAQ;AAAA,MAC5C,CAAC;AACD,WAAK;AAAA,QACH,0BAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,4BAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,0BAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,4BAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oCAAoB;AACxC,cAAQ,SAAS,4BAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAQ;AAAA,QAC5C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAQ;AAC1C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AA5OnE,YAAAC;AA8OQ,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AAhQ9D,YAAAA,KAAA;AAkQQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AA5Q5E,YAAAA,KAAA;AA8QQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AA9R5D,YAAAA,KAAA;AA+RQ,aAAK,KAAK,uBAAuB;AACjC,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,WAAK,SAAS,GAAG,qBAAqB,CAAC,YAAkC;AACvE,aAAK,KAAK,qBAAqB,OAAO;AAAA,MACxC,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AArYJ;AAsYI,QACE,YAAY,WAAW,4BAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,SAAK,gBAAgB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC1D,0BAAoB,IAAI,4BAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,oBAAmC;AAnarC;AAoaI,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,qBAAiB,+BAAiB,KAAK,OAAM,UAAK,KAAK,qBAAV,mBAA4B,QAAQ;AAAA,IACxF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AAhbV;AAibI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AAtd/B;AAudI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAW,qBAAqB;AAChF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}

package/dist/multimodal/multimodal_agent.d.ts

@@ -32,10 +32,7 @@ export declare class MultimodalAgent extends EventEmitter {
     room: Room | null;
     linkedParticipant: RemoteParticipant | null;
     subscribedTrack: RemoteAudioTrack | null;
-    readMicroTask: {
-        promise: Promise<void>;
-        cancel: () => void;
-    } | null;
+    readMicroTask: Promise<void> | null;
     constructor({ model, chatCtx, fncCtx, }: {
         model: RealtimeModel;
         chatCtx?: llm.ChatContext;

package/dist/multimodal/multimodal_agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"multimodal_agent.d.ts","sourceRoot":"","sources":["../../src/multimodal/multimodal_agent.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAEV,gBAAgB,EAChB,iBAAiB,EAGjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,GAAG,MAAM,iBAAiB,CAAC;AAMvC;;;GAGG;AACH,8BAAsB,eAAgB,SAAQ,YAAY;IAExD,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC;IAE3B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;CAClD;AAED;;;GAGG;AACH,8BAAsB,aAAa;IAEjC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,eAAe;IAC/C,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD,YAAY;AACZ,qBAAa,eAAgB,SAAQ,YAAY;;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,IAAI,GAAG,IAAI,CAAQ;IACzB,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAQ;IACnD,eAAe,EAAE,gBAAgB,GAAG,IAAI,CAAQ;IAChD,aAAa,EAAE;QAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAAC,MAAM,EAAE,MAAM,IAAI,CAAA;KAAE,GAAG,IAAI,CAAQ;gBAEhE,EACV,KAAK,EACL,OAAO,EACP,MAAM,GACP,EAAE;QACD,KAAK,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;KAC9B;IAsBD,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAK9C;IA6BD,KAAK,CACH,IAAI,EAAE,IAAI,EACV,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW,GACpD,OAAO,CAAC,eAAe,CAAC;CA+V5B"}
+{"version":3,"file":"multimodal_agent.d.ts","sourceRoot":"","sources":["../../src/multimodal/multimodal_agent.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAEV,gBAAgB,EAChB,iBAAiB,EAGjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,GAAG,MAAM,iBAAiB,CAAC;AAOvC;;;GAGG;AACH,8BAAsB,eAAgB,SAAQ,YAAY;IAExD,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC;IAE3B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;CAClD;AAED;;;GAGG;AACH,8BAAsB,aAAa;IAEjC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,eAAe;IAC/C,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD,YAAY;AACZ,qBAAa,eAAgB,SAAQ,YAAY;;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,IAAI,GAAG,IAAI,CAAQ;IACzB,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAQ;IACnD,eAAe,EAAE,gBAAgB,GAAG,IAAI,CAAQ;IAChD,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;gBAE/B,EACV,KAAK,EACL,OAAO,EACP,MAAM,GACP,EAAE;QACD,KAAK,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;KAC9B;IAsBD,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAK9C;IA6BD,KAAK,CACH,IAAI,EAAE,IAAI,EACV,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW,GACpD,OAAO,CAAC,eAAe,CAAC;CAyV5B"}

package/dist/multimodal/multimodal_agent.js

@@ -205,6 +205,7 @@ class MultimodalAgent extends EventEmitter {
       });
       this.#session.on("input_speech_started", (ev) => {
         var _a2, _b;
+        this.emit("user_started_speaking");
         if (this.#playingHandle && !this.#playingHandle.done) {
           this.#playingHandle.interrupt();
           this.#session.conversation.item.truncate(
@@ -235,6 +236,9 @@ class MultimodalAgent extends EventEmitter {
         this.#pendingFunctionCalls.delete(ev.callId);
         this.#updateState();
       });
+      this.#session.on("metrics_collected", (metrics) => {
+        this.emit("metrics_collected", metrics);
+      });
       resolve(this.#session);
     });
   }
@@ -296,19 +300,9 @@ class MultimodalAgent extends EventEmitter {
       }
     };
     this.subscribedTrack = track;
-    if (this.readMicroTask) {
-      this.readMicroTask.cancel();
-    }
-    let cancel;
-    this.readMicroTask = {
-      promise: new Promise((resolve, reject) => {
-        cancel = () => {
-          reject(new Error("Task cancelled"));
-        };
-        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
-      }),
-      cancel: () => cancel()
-    };
+    this.readMicroTask = new Promise((resolve, reject) => {
+      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
+    });
   }
   #getLocalTrackSid() {
     var _a;