@livekit/agents 0.4.6 → 0.5.1

package/dist/pipeline/agent_playout.cjs

@@ -0,0 +1,169 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var agent_playout_exports = {};
+__export(agent_playout_exports, {
+  AgentPlayout: () => AgentPlayout,
+  AgentPlayoutEvent: () => AgentPlayoutEvent,
+  PlayoutHandle: () => PlayoutHandle
+});
+module.exports = __toCommonJS(agent_playout_exports);
+var import_node_events = __toESM(require("node:events"), 1);
+var import_log = require("../log.cjs");
+var import_utils = require("../utils.cjs");
+var import_agent_output = require("./agent_output.cjs");
+var AgentPlayoutEvent = /* @__PURE__ */ ((AgentPlayoutEvent2) => {
+  AgentPlayoutEvent2[AgentPlayoutEvent2["PLAYOUT_STARTED"] = 0] = "PLAYOUT_STARTED";
+  AgentPlayoutEvent2[AgentPlayoutEvent2["PLAYOUT_STOPPED"] = 1] = "PLAYOUT_STOPPED";
+  return AgentPlayoutEvent2;
+})(AgentPlayoutEvent || {});
+class PlayoutHandle {
+  #speechId;
+  #audioSource;
+  playoutSource;
+  totalPlayedTime;
+  #interrupted = false;
+  pushedDuration = 0;
+  intFut = new import_utils.Future();
+  doneFut = new import_utils.Future();
+  constructor(speechId, audioSource, playoutSource) {
+    this.#speechId = speechId;
+    this.#audioSource = audioSource;
+    this.playoutSource = playoutSource;
+  }
+  get speechId() {
+    return this.#speechId;
+  }
+  get interrupted() {
+    return this.#interrupted;
+  }
+  get timePlayed() {
+    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;
+  }
+  get done() {
+    return this.doneFut.done || this.#interrupted;
+  }
+  interrupt() {
+    if (this.done) {
+      return;
+    }
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+  join() {
+    return this.doneFut;
+  }
+}
+class AgentPlayout extends import_node_events.default {
+  #closed = false;
+  #audioSource;
+  #targetVolume = 1;
+  #playoutTask;
+  #logger = (0, import_log.log)();
+  constructor(audioSource) {
+    super();
+    this.#audioSource = audioSource;
+  }
+  get targetVolume() {
+    return this.#targetVolume;
+  }
+  set targetVolume(vol) {
+    this.#targetVolume = vol;
+  }
+  play(speechId, playoutSource) {
+    if (this.#closed) {
+      throw new Error("source closed");
+    }
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+    this.#playoutTask = this.#playout(handle, this.#playoutTask);
+    return handle;
+  }
+  #playout(handle, oldTask) {
+    return new import_utils.CancellablePromise(async (resolve, _, onCancel) => {
+      const cancel = () => {
+        captureTask.cancel();
+        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+        if (handle.interrupted || captureTask.error) {
+          this.#audioSource.clearQueue();
+        }
+        if (!firstFrame) {
+          this.emit(1 /* PLAYOUT_STOPPED */, handle.interrupted);
+        }
+        handle.doneFut.resolve();
+        this.#logger.child({ speechId: handle.speechId, interrupted: handle.interrupted }).debug("playout finished");
+      };
+      onCancel(() => {
+        cancel();
+      });
+      if (oldTask) {
+        await (0, import_utils.gracefullyCancel)(oldTask);
+      }
+      if (this.#audioSource.queuedDuration > 0) {
+        this.#logger.child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration }).warn("new playout while the source is still playing");
+      }
+      let firstFrame = true;
+      const captureTask = new import_utils.CancellablePromise(async (resolve2, _2, onCancel2) => {
+        let cancelled = false;
+        onCancel2(() => {
+          cancelled = true;
+        });
+        for await (const frame of handle.playoutSource) {
+          if (cancelled || frame === import_agent_output.SynthesisHandle.FLUSH_SENTINEL) {
+            break;
+          }
+          if (firstFrame) {
+            this.#logger.child({ speechId: handle.speechId }).debug("started playing the first time");
+            this.emit(0 /* PLAYOUT_STARTED */);
+            firstFrame = false;
+          }
+          handle.pushedDuration += frame.samplesPerChannel / frame.sampleRate * 1e3;
+          await this.#audioSource.captureFrame(frame);
+          await this.#audioSource.waitForPlayout();
+        }
+        resolve2();
+      });
+      try {
+        await Promise.any([captureTask, handle.intFut.await]);
+      } finally {
+        cancel();
+        resolve();
+      }
+    });
+  }
+  async close() {
+    this.#closed = true;
+    await this.#playoutTask;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AgentPlayout,
+  AgentPlayoutEvent,
+  PlayoutHandle
+});
+//# sourceMappingURL=agent_playout.cjs.map

package/dist/pipeline/agent_playout.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/pipeline/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, AudioSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport { log } from '../log.js';\nimport { CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { SynthesisHandle } from './agent_output.js';\n\nexport enum AgentPlayoutEvent {\n  PLAYOUT_STARTED,\n  PLAYOUT_STOPPED,\n}\n\nexport type AgentPlayoutCallbacks = {\n  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;\n  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;\n};\n\nexport class PlayoutHandle {\n  #speechId: string;\n  #audioSource: AudioSource;\n  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;\n  totalPlayedTime?: number;\n  #interrupted = false;\n  pushedDuration = 0;\n  intFut = new Future();\n  doneFut = new Future();\n\n  constructor(\n    speechId: string,\n    audioSource: AudioSource,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ) {\n    this.#speechId = speechId;\n    this.#audioSource = audioSource;\n    this.playoutSource = playoutSource;\n  }\n\n  get speechId(): string {\n    return this.#speechId;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get timePlayed(): number {\n    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.done) {\n      return;\n    }\n\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n\n  join(): Future {\n    return this.doneFut;\n  }\n}\n\nexport class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {\n  #closed = false;\n  #audioSource: AudioSource;\n  #targetVolume = 1;\n  #playoutTask?: CancellablePromise<void>;\n  #logger = log();\n\n  constructor(audioSource: AudioSource) {\n    super();\n    this.#audioSource = audioSource;\n  }\n\n  get targetVolume(): number {\n    return this.#targetVolume;\n  }\n\n  set targetVolume(vol: number) {\n    this.#targetVolume = vol;\n  }\n\n  play(\n    speechId: string,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ): PlayoutHandle {\n    if (this.#closed) {\n      throw new Error('source closed');\n    }\n\n    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);\n\n    this.#playoutTask = this.#playout(handle, this.#playoutTask);\n    return handle;\n  }\n\n  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      const cancel = () => {\n        captureTask.cancel();\n        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n        if (handle.interrupted || captureTask.error) {\n          this.#audioSource.clearQueue(); // make sure to remove any queued frames\n        }\n\n        if (!firstFrame) {\n          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);\n        }\n\n        handle.doneFut.resolve();\n\n        this.#logger\n          .child({ speechId: handle.speechId, interrupted: handle.interrupted })\n          .debug('playout finished');\n      };\n\n      onCancel(() => {\n        cancel();\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      if (this.#audioSource.queuedDuration > 0) {\n        // this should not happen, but log it just in case\n        this.#logger\n          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })\n          .warn('new playout while the source is still playing');\n      }\n\n      let firstFrame = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        for await (const frame of handle.playoutSource) {\n          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {\n            break;\n          }\n          if (firstFrame) {\n            this.#logger\n              .child({ speechId: handle.speechId })\n              .debug('started playing the first time');\n            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);\n            firstFrame = false;\n          }\n          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;\n          await this.#audioSource.captureFrame(frame);\n          await this.#audioSource.waitForPlayout();\n        }\n\n        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,\n        //            but for some reason too many TTS frames can gunk up the buffer and lead to\n        //            FFI errors. this works 🤷‍♀️\n        // if (this.#audioSource.queuedDuration > 0) {\n        //   await this.#audioSource.waitForPlayout();\n        // }\n\n        resolve();\n      });\n\n      try {\n        await Promise.any([captureTask, handle.intFut.await]);\n      } finally {\n        cancel();\n        resolve();\n      }\n    });\n  }\n\n  async close() {\n    this.#closed = true;\n    await this.#playoutTask;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAAyB;AACzB,iBAAoB;AACpB,mBAA6D;AAC7D,0BAAgC;AAEzB,IAAK,oBAAL,kBAAKA,uBAAL;AACL,EAAAA,sCAAA;AACA,EAAAA,sCAAA;AAFU,SAAAA;AAAA,GAAA;AAUL,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,SAAS,IAAI,oBAAO;AAAA,EACpB,UAAU,IAAI,oBAAO;AAAA,EAErB,YACE,UACA,aACA,eACA;AACA,SAAK,YAAY;AACjB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK,mBAAmB,KAAK,iBAAiB,KAAK,aAAa;AAAA,EACzE;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,MAAM;AACb;AAAA,IACF;AAEA,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,OAAe;AACb,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,qBAAsB,mBAAAC,QAA+D;AAAA,EAChG,UAAU;AAAA,EACV;AAAA,EACA,gBAAgB;AAAA,EAChB;AAAA,EACA,cAAU,gBAAI;AAAA,EAEd,YAAY,aAA0B;AACpC,UAAM;AACN,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAa,KAAa;AAC5B,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,UACA,eACe;AACf,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,eAAe;AAAA,IACjC;AAEA,UAAM,SAAS,IAAI,cAAc,UAAU,KAAK,cAAc,aAAa;AAE3E,SAAK,eAAe,KAAK,SAAS,QAAQ,KAAK,YAAY;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,QAAuB,SAA8D;AAC5F,WAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,YAAM,SAAS,MAAM;AACnB,oBAAY,OAAO;AACnB,eAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,YAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,eAAK,aAAa,WAAW;AAAA,QAC/B;AAEA,YAAI,CAAC,YAAY;AACf,eAAK,KAAK,yBAAmC,OAAO,WAAW;AAAA,QACjE;AAEA,eAAO,QAAQ,QAAQ;AAEvB,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,aAAa,OAAO,YAAY,CAAC,EACpE,MAAM,kBAAkB;AAAA,MAC7B;AAEA,eAAS,MAAM;AACb,eAAO;AAAA,MACT,CAAC;AAED,UAAI,SAAS;AACX,kBAAM,+BAAiB,OAAO;AAAA,MAChC;AAEA,UAAI,KAAK,aAAa,iBAAiB,GAAG;AAExC,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,gBAAgB,KAAK,aAAa,eAAe,CAAC,EACrF,KAAK,+CAA+C;AAAA,MACzD;AAEA,UAAI,aAAa;AAGjB,YAAM,cAAc,IAAI,gCAAyB,OAAOC,UAASC,IAAGC,cAAa;AAC/E,YAAI,YAAY;AAChB,QAAAA,UAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,yBAAiB,SAAS,OAAO,eAAe;AAC9C,cAAI,aAAa,UAAU,oCAAgB,gBAAgB;AACzD;AAAA,UACF;AACA,cAAI,YAAY;AACd,iBAAK,QACF,MAAM,EAAE,UAAU,OAAO,SAAS,CAAC,EACnC,MAAM,gCAAgC;AACzC,iBAAK,KAAK,uBAAiC;AAC3C,yBAAa;AAAA,UACf;AACA,iBAAO,kBAAmB,MAAM,oBAAoB,MAAM,aAAc;AACxE,gBAAM,KAAK,aAAa,aAAa,KAAK;AAC1C,gBAAM,KAAK,aAAa,eAAe;AAAA,QACzC;AASA,QAAAF,SAAQ;AAAA,MACV,CAAC;AAED,UAAI;AACF,cAAM,QAAQ,IAAI,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,MACtD,UAAE;AACA,eAAO;AACP,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AACF;","names":["AgentPlayoutEvent","EventEmitter","resolve","_","onCancel"]}

package/dist/pipeline/agent_playout.js

@@ -1,143 +1,133 @@
-import EventEmitter from 'node:events';
-import { log } from '../log.js';
-import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
-import { SynthesisHandle } from './agent_output.js';
-export var AgentPlayoutEvent;
-(function (AgentPlayoutEvent) {
-    AgentPlayoutEvent[AgentPlayoutEvent["PLAYOUT_STARTED"] = 0] = "PLAYOUT_STARTED";
-    AgentPlayoutEvent[AgentPlayoutEvent["PLAYOUT_STOPPED"] = 1] = "PLAYOUT_STOPPED";
-})(AgentPlayoutEvent || (AgentPlayoutEvent = {}));
-export class PlayoutHandle {
-    #speechId;
-    #audioSource;
-    playoutSource;
-    totalPlayedTime;
-    #interrupted = false;
-    pushedDuration = 0;
-    intFut = new Future();
-    doneFut = new Future();
-    constructor(speechId, audioSource, playoutSource) {
-        this.#speechId = speechId;
-        this.#audioSource = audioSource;
-        this.playoutSource = playoutSource;
-    }
-    get speechId() {
-        return this.#speechId;
-    }
-    get interrupted() {
-        return this.#interrupted;
-    }
-    get timePlayed() {
-        return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;
-    }
-    get done() {
-        return this.doneFut.done || this.#interrupted;
-    }
-    interrupt() {
-        if (this.done) {
-            return;
-        }
-        this.intFut.resolve();
-        this.#interrupted = true;
-    }
-    join() {
-        return this.doneFut;
+import EventEmitter from "node:events";
+import { log } from "../log.js";
+import { CancellablePromise, Future, gracefullyCancel } from "../utils.js";
+import { SynthesisHandle } from "./agent_output.js";
+var AgentPlayoutEvent = /* @__PURE__ */ ((AgentPlayoutEvent2) => {
+  AgentPlayoutEvent2[AgentPlayoutEvent2["PLAYOUT_STARTED"] = 0] = "PLAYOUT_STARTED";
+  AgentPlayoutEvent2[AgentPlayoutEvent2["PLAYOUT_STOPPED"] = 1] = "PLAYOUT_STOPPED";
+  return AgentPlayoutEvent2;
+})(AgentPlayoutEvent || {});
+class PlayoutHandle {
+  #speechId;
+  #audioSource;
+  playoutSource;
+  totalPlayedTime;
+  #interrupted = false;
+  pushedDuration = 0;
+  intFut = new Future();
+  doneFut = new Future();
+  constructor(speechId, audioSource, playoutSource) {
+    this.#speechId = speechId;
+    this.#audioSource = audioSource;
+    this.playoutSource = playoutSource;
+  }
+  get speechId() {
+    return this.#speechId;
+  }
+  get interrupted() {
+    return this.#interrupted;
+  }
+  get timePlayed() {
+    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;
+  }
+  get done() {
+    return this.doneFut.done || this.#interrupted;
+  }
+  interrupt() {
+    if (this.done) {
+      return;
     }
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+  join() {
+    return this.doneFut;
+  }
 }
-export class AgentPlayout extends EventEmitter {
-    #closed = false;
-    #audioSource;
-    #targetVolume = 1;
-    #playoutTask;
-    #logger = log();
-    constructor(audioSource) {
-        super();
-        this.#audioSource = audioSource;
+class AgentPlayout extends EventEmitter {
+  #closed = false;
+  #audioSource;
+  #targetVolume = 1;
+  #playoutTask;
+  #logger = log();
+  constructor(audioSource) {
+    super();
+    this.#audioSource = audioSource;
+  }
+  get targetVolume() {
+    return this.#targetVolume;
+  }
+  set targetVolume(vol) {
+    this.#targetVolume = vol;
+  }
+  play(speechId, playoutSource) {
+    if (this.#closed) {
+      throw new Error("source closed");
     }
-    get targetVolume() {
-        return this.#targetVolume;
-    }
-    set targetVolume(vol) {
-        this.#targetVolume = vol;
-    }
-    play(speechId, playoutSource) {
-        if (this.#closed) {
-            throw new Error('source closed');
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+    this.#playoutTask = this.#playout(handle, this.#playoutTask);
+    return handle;
+  }
+  #playout(handle, oldTask) {
+    return new CancellablePromise(async (resolve, _, onCancel) => {
+      const cancel = () => {
+        captureTask.cancel();
+        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+        if (handle.interrupted || captureTask.error) {
+          this.#audioSource.clearQueue();
         }
-        const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
-        this.#playoutTask = this.#playout(handle, this.#playoutTask);
-        return handle;
-    }
-    #playout(handle, oldTask) {
-        return new CancellablePromise(async (resolve, _, onCancel) => {
-            const cancel = () => {
-                captureTask.cancel();
-                handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
-                if (handle.interrupted || captureTask.error) {
-                    this.#audioSource.clearQueue(); // make sure to remove any queued frames
-                }
-                if (!firstFrame) {
-                    this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);
-                }
-                handle.doneFut.resolve();
-                this.#logger
-                    .child({ speechId: handle.speechId, interrupted: handle.interrupted })
-                    .debug('playout finished');
-            };
-            onCancel(() => {
-                cancel();
-            });
-            if (oldTask) {
-                await gracefullyCancel(oldTask);
-            }
-            if (this.#audioSource.queuedDuration > 0) {
-                // this should not happen, but log it just in case
-                this.#logger
-                    .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })
-                    .warn('new playout while the source is still playing');
-            }
-            let firstFrame = true;
-            // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            const captureTask = new CancellablePromise(async (resolve, _, onCancel) => {
-                let cancelled = false;
-                onCancel(() => {
-                    cancelled = true;
-                });
-                for await (const frame of handle.playoutSource) {
-                    if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {
-                        break;
-                    }
-                    if (firstFrame) {
-                        this.#logger
-                            .child({ speechId: handle.speechId })
-                            .debug('started playing the first time');
-                        this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);
-                        firstFrame = false;
-                    }
-                    handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
-                    await this.#audioSource.captureFrame(frame);
-                    await this.#audioSource.waitForPlayout();
-                }
-                // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,
-                //            but for some reason too many TTS frames can gunk up the buffer and lead to
-                //            FFI errors. this works 🤷‍♀️
-                // if (this.#audioSource.queuedDuration > 0) {
-                //   await this.#audioSource.waitForPlayout();
-                // }
-                resolve();
-            });
-            try {
-                await Promise.any([captureTask, handle.intFut.await]);
-            }
-            finally {
-                cancel();
-                resolve();
-            }
+        if (!firstFrame) {
+          this.emit(1 /* PLAYOUT_STOPPED */, handle.interrupted);
+        }
+        handle.doneFut.resolve();
+        this.#logger.child({ speechId: handle.speechId, interrupted: handle.interrupted }).debug("playout finished");
+      };
+      onCancel(() => {
+        cancel();
+      });
+      if (oldTask) {
+        await gracefullyCancel(oldTask);
+      }
+      if (this.#audioSource.queuedDuration > 0) {
+        this.#logger.child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration }).warn("new playout while the source is still playing");
+      }
+      let firstFrame = true;
+      const captureTask = new CancellablePromise(async (resolve2, _2, onCancel2) => {
+        let cancelled = false;
+        onCancel2(() => {
+          cancelled = true;
         });
-    }
-    async close() {
-        this.#closed = true;
-        await this.#playoutTask;
-    }
+        for await (const frame of handle.playoutSource) {
+          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {
+            break;
+          }
+          if (firstFrame) {
+            this.#logger.child({ speechId: handle.speechId }).debug("started playing the first time");
+            this.emit(0 /* PLAYOUT_STARTED */);
+            firstFrame = false;
+          }
+          handle.pushedDuration += frame.samplesPerChannel / frame.sampleRate * 1e3;
+          await this.#audioSource.captureFrame(frame);
+          await this.#audioSource.waitForPlayout();
+        }
+        resolve2();
+      });
+      try {
+        await Promise.any([captureTask, handle.intFut.await]);
+      } finally {
+        cancel();
+        resolve();
+      }
+    });
+  }
+  async close() {
+    this.#closed = true;
+    await this.#playoutTask;
+  }
 }
+export {
+  AgentPlayout,
+  AgentPlayoutEvent,
+  PlayoutHandle
+};
 //# sourceMappingURL=agent_playout.js.map

package/dist/pipeline/agent_playout.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent_playout.js","sourceRoot":"","sources":["../../src/pipeline/agent_playout.ts"],"names":[],"mappings":"AAKA,OAAO,YAAY,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChC,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,MAAM,CAAN,IAAY,iBAGX;AAHD,WAAY,iBAAiB;IAC3B,+EAAe,CAAA;IACf,+EAAe,CAAA;AACjB,CAAC,EAHW,iBAAiB,KAAjB,iBAAiB,QAG5B;AAOD,MAAM,OAAO,aAAa;IACxB,SAAS,CAAS;IAClB,YAAY,CAAc;IAC1B,aAAa,CAAoE;IACjF,eAAe,CAAU;IACzB,YAAY,GAAG,KAAK,CAAC;IACrB,cAAc,GAAG,CAAC,CAAC;IACnB,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IACtB,OAAO,GAAG,IAAI,MAAM,EAAE,CAAC;IAEvB,YACE,QAAgB,EAChB,WAAwB,EACxB,aAAgF;QAEhF,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC;IACxF,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC;IAChD,CAAC;IAED,SAAS;QACP,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAS,YAA8D;IAC/F,OAAO,GAAG,KAAK,CAAC;IAChB,YAAY,CAAc;IAC1B,aAAa,GAAG,CAAC,CAAC;IAClB,YAAY,CAA4B;IACxC,OAAO,GAAG,GAAG,EAAE,CAAC;IAEhB,YAAY,WAAwB;QAClC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,IAAI,YAAY,CAAC,GAAW;QAC1B,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;IAC3B,CAAC;IAED,IAAI,CACF,QAAgB,EAChB,aAAgF;QAEhF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAE7E,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,QAAQ,CAAC,MAAqB,EAAE,OAAkC;QAChE,OAAO,IAAI,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC3D,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,WAAW,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC;gBAElF,IAAI,MAAM,CAAC,WAAW,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;oBAC5C,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,wCAAwC;gBAC1E,CAAC;gBAED,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;gBACnE,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAEzB,IAAI,CAAC,OAAO;qBACT,KAAK,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;qBACrE,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAC/B,CAAC,CAAC;YAEF,QAAQ,CAAC,GAAG,EAAE;gBACZ,MAAM,EAAE,CAAC;YACX,CAAC,CAAC,CAAC;YAEH,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBACzC,kDAAkD;gBAClD,IAAI,CAAC,OAAO;qBACT,KAAK,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;qBACtF,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC3D,CAAC;YAED,IAAI,UAAU,GAAG,IAAI,CAAC;YAEtB,6DAA6D;YAC7D,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAO,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;gBAC9E,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,QAAQ,CAAC,GAAG,EAAE;oBACZ,SAAS,GAAG,IAAI,CAAC;gBACnB,CAAC,CAAC,CAAC;gBAEH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;oBAC/C,IAAI,SAAS,IAAI,KAAK,KAAK,eAAe,CAAC,cAAc,EAAE,CAAC;wBAC1D,MAAM;oBACR,CAAC;oBACD,IAAI,UAAU,EAAE,CAAC;wBACf,IAAI,CAAC,OAAO;6BACT,KAAK,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;6BACpC,KAAK,CAAC,gCAAgC,CAAC,CAAC;wBAC3C,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;wBAC7C,UAAU,GAAG,KAAK,CAAC;oBACrB,CAAC;oBACD,MAAM,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;oBAC7E,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;gBAC3C,CAAC;gBAED,oFAAoF;gBACpF,wFAAwF;gBACxF,0CAA0C;gBAC1C,8CAA8C;gBAC9C,8CAA8C;gBAC9C,IAAI;gBAEJ,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,CAAC;oBAAS,CAAC;gBACT,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,MAAM,IAAI,CAAC,YAAY,CAAC;IAC1B,CAAC;CACF"}
+{"version":3,"sources":["../../src/pipeline/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, AudioSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport { log } from '../log.js';\nimport { CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { SynthesisHandle } from './agent_output.js';\n\nexport enum AgentPlayoutEvent {\n  PLAYOUT_STARTED,\n  PLAYOUT_STOPPED,\n}\n\nexport type AgentPlayoutCallbacks = {\n  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;\n  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;\n};\n\nexport class PlayoutHandle {\n  #speechId: string;\n  #audioSource: AudioSource;\n  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;\n  totalPlayedTime?: number;\n  #interrupted = false;\n  pushedDuration = 0;\n  intFut = new Future();\n  doneFut = new Future();\n\n  constructor(\n    speechId: string,\n    audioSource: AudioSource,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ) {\n    this.#speechId = speechId;\n    this.#audioSource = audioSource;\n    this.playoutSource = playoutSource;\n  }\n\n  get speechId(): string {\n    return this.#speechId;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get timePlayed(): number {\n    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.done) {\n      return;\n    }\n\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n\n  join(): Future {\n    return this.doneFut;\n  }\n}\n\nexport class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {\n  #closed = false;\n  #audioSource: AudioSource;\n  #targetVolume = 1;\n  #playoutTask?: CancellablePromise<void>;\n  #logger = log();\n\n  constructor(audioSource: AudioSource) {\n    super();\n    this.#audioSource = audioSource;\n  }\n\n  get targetVolume(): number {\n    return this.#targetVolume;\n  }\n\n  set targetVolume(vol: number) {\n    this.#targetVolume = vol;\n  }\n\n  play(\n    speechId: string,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ): PlayoutHandle {\n    if (this.#closed) {\n      throw new Error('source closed');\n    }\n\n    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);\n\n    this.#playoutTask = this.#playout(handle, this.#playoutTask);\n    return handle;\n  }\n\n  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      const cancel = () => {\n        captureTask.cancel();\n        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n        if (handle.interrupted || captureTask.error) {\n          this.#audioSource.clearQueue(); // make sure to remove any queued frames\n        }\n\n        if (!firstFrame) {\n          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);\n        }\n\n        handle.doneFut.resolve();\n\n        this.#logger\n          .child({ speechId: handle.speechId, interrupted: handle.interrupted })\n          .debug('playout finished');\n      };\n\n      onCancel(() => {\n        cancel();\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      if (this.#audioSource.queuedDuration > 0) {\n        // this should not happen, but log it just in case\n        this.#logger\n          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })\n          .warn('new playout while the source is still playing');\n      }\n\n      let firstFrame = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        for await (const frame of handle.playoutSource) {\n          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {\n            break;\n          }\n          if (firstFrame) {\n            this.#logger\n              .child({ speechId: handle.speechId })\n              .debug('started playing the first time');\n            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);\n            firstFrame = false;\n          }\n          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;\n          await this.#audioSource.captureFrame(frame);\n          await this.#audioSource.waitForPlayout();\n        }\n\n        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,\n        //            but for some reason too many TTS frames can gunk up the buffer and lead to\n        //            FFI errors. this works 🤷‍♀️\n        // if (this.#audioSource.queuedDuration > 0) {\n        //   await this.#audioSource.waitForPlayout();\n        // }\n\n        resolve();\n      });\n\n      try {\n        await Promise.any([captureTask, handle.intFut.await]);\n      } finally {\n        cancel();\n        resolve();\n      }\n    });\n  }\n\n  async close() {\n    this.#closed = true;\n    await this.#playoutTask;\n  }\n}\n"],"mappings":"AAKA,OAAO,kBAAkB;AACzB,SAAS,WAAW;AACpB,SAAS,oBAAoB,QAAQ,wBAAwB;AAC7D,SAAS,uBAAuB;AAEzB,IAAK,oBAAL,kBAAKA,uBAAL;AACL,EAAAA,sCAAA;AACA,EAAAA,sCAAA;AAFU,SAAAA;AAAA,GAAA;AAUL,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,SAAS,IAAI,OAAO;AAAA,EACpB,UAAU,IAAI,OAAO;AAAA,EAErB,YACE,UACA,aACA,eACA;AACA,SAAK,YAAY;AACjB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK,mBAAmB,KAAK,iBAAiB,KAAK,aAAa;AAAA,EACzE;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,MAAM;AACb;AAAA,IACF;AAEA,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,OAAe;AACb,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,qBAAsB,aAA+D;AAAA,EAChG,UAAU;AAAA,EACV;AAAA,EACA,gBAAgB;AAAA,EAChB;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,YAAY,aAA0B;AACpC,UAAM;AACN,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAa,KAAa;AAC5B,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,UACA,eACe;AACf,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,eAAe;AAAA,IACjC;AAEA,UAAM,SAAS,IAAI,cAAc,UAAU,KAAK,cAAc,aAAa;AAE3E,SAAK,eAAe,KAAK,SAAS,QAAQ,KAAK,YAAY;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,QAAuB,SAA8D;AAC5F,WAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,YAAM,SAAS,MAAM;AACnB,oBAAY,OAAO;AACnB,eAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,YAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,eAAK,aAAa,WAAW;AAAA,QAC/B;AAEA,YAAI,CAAC,YAAY;AACf,eAAK,KAAK,yBAAmC,OAAO,WAAW;AAAA,QACjE;AAEA,eAAO,QAAQ,QAAQ;AAEvB,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,aAAa,OAAO,YAAY,CAAC,EACpE,MAAM,kBAAkB;AAAA,MAC7B;AAEA,eAAS,MAAM;AACb,eAAO;AAAA,MACT,CAAC;AAED,UAAI,SAAS;AACX,cAAM,iBAAiB,OAAO;AAAA,MAChC;AAEA,UAAI,KAAK,aAAa,iBAAiB,GAAG;AAExC,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,gBAAgB,KAAK,aAAa,eAAe,CAAC,EACrF,KAAK,+CAA+C;AAAA,MACzD;AAEA,UAAI,aAAa;AAGjB,YAAM,cAAc,IAAI,mBAAyB,OAAOC,UAASC,IAAGC,cAAa;AAC/E,YAAI,YAAY;AAChB,QAAAA,UAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,yBAAiB,SAAS,OAAO,eAAe;AAC9C,cAAI,aAAa,UAAU,gBAAgB,gBAAgB;AACzD;AAAA,UACF;AACA,cAAI,YAAY;AACd,iBAAK,QACF,MAAM,EAAE,UAAU,OAAO,SAAS,CAAC,EACnC,MAAM,gCAAgC;AACzC,iBAAK,KAAK,uBAAiC;AAC3C,yBAAa;AAAA,UACf;AACA,iBAAO,kBAAmB,MAAM,oBAAoB,MAAM,aAAc;AACxE,gBAAM,KAAK,aAAa,aAAa,KAAK;AAC1C,gBAAM,KAAK,aAAa,eAAe;AAAA,QACzC;AASA,QAAAF,SAAQ;AAAA,MACV,CAAC;AAED,UAAI;AACF,cAAM,QAAQ,IAAI,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,MACtD,UAAE;AACA,eAAO;AACP,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AACF;","names":["AgentPlayoutEvent","resolve","_","onCancel"]}

package/dist/pipeline/human_input.cjs

@@ -0,0 +1,158 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var human_input_exports = {};
+__export(human_input_exports, {
+  HumanInput: () => HumanInput,
+  HumanInputEvent: () => HumanInputEvent
+});
+module.exports = __toCommonJS(human_input_exports);
+var import_rtc_node = require("@livekit/rtc-node");
+var import_node_events = require("node:events");
+var import_log = require("../log.cjs");
+var import_stt = require("../stt/stt.cjs");
+var import_utils = require("../utils.cjs");
+var import_vad = require("../vad.cjs");
+var HumanInputEvent = /* @__PURE__ */ ((HumanInputEvent2) => {
+  HumanInputEvent2[HumanInputEvent2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
+  HumanInputEvent2[HumanInputEvent2["VAD_INFERENCE_DONE"] = 1] = "VAD_INFERENCE_DONE";
+  HumanInputEvent2[HumanInputEvent2["END_OF_SPEECH"] = 2] = "END_OF_SPEECH";
+  HumanInputEvent2[HumanInputEvent2["FINAL_TRANSCRIPT"] = 3] = "FINAL_TRANSCRIPT";
+  HumanInputEvent2[HumanInputEvent2["INTERIM_TRANSCRIPT"] = 4] = "INTERIM_TRANSCRIPT";
+  return HumanInputEvent2;
+})(HumanInputEvent || {});
+class HumanInput extends import_node_events.EventEmitter {
+  #closed = false;
+  #room;
+  #vad;
+  #stt;
+  #participant;
+  #subscribedTrack;
+  #recognizeTask;
+  #speaking = false;
+  #speechProbability = 0;
+  #logger = (0, import_log.log)();
+  constructor(room, vad, stt, participant) {
+    super();
+    this.#room = room;
+    this.#vad = vad;
+    this.#stt = stt;
+    this.#participant = participant;
+    this.#room.on(import_rtc_node.RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));
+    this.#room.on(import_rtc_node.RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));
+    this.#subscribeToMicrophone();
+  }
+  #subscribeToMicrophone() {
+    if (!this.#participant) {
+      this.#logger.error("Participant is not set");
+      return;
+    }
+    let microphonePublication = void 0;
+    for (const publication of this.#participant.trackPublications.values()) {
+      if (publication.source === import_rtc_node.TrackSource.SOURCE_MICROPHONE) {
+        microphonePublication = publication;
+        break;
+      }
+    }
+    if (!microphonePublication) {
+      return;
+    }
+    if (!microphonePublication.subscribed) {
+      microphonePublication.setSubscribed(true);
+    }
+    const track = microphonePublication.track;
+    if (track && track !== this.#subscribedTrack) {
+      this.#subscribedTrack = track;
+      if (this.#recognizeTask) {
+        this.#recognizeTask.cancel();
+      }
+      const audioStream = new import_rtc_node.AudioStream(track, 16e3);
+      this.#recognizeTask = new import_utils.CancellablePromise(async (resolve, _, onCancel) => {
+        let cancelled = false;
+        onCancel(() => {
+          cancelled = true;
+        });
+        const sttStream = this.#stt.stream();
+        const vadStream = this.#vad.stream();
+        const audioStreamCo = async () => {
+          for await (const ev of audioStream) {
+            if (cancelled) return;
+            sttStream.pushFrame(ev);
+            vadStream.pushFrame(ev);
+          }
+        };
+        const vadStreamCo = async () => {
+          for await (const ev of vadStream) {
+            if (cancelled) return;
+            switch (ev.type) {
+              case import_vad.VADEventType.START_OF_SPEECH:
+                this.#speaking = true;
+                this.emit(0 /* START_OF_SPEECH */, ev);
+                break;
+              case import_vad.VADEventType.INFERENCE_DONE:
+                this.#speechProbability = ev.probability;
+                this.emit(1 /* VAD_INFERENCE_DONE */, ev);
+                break;
+              case import_vad.VADEventType.END_OF_SPEECH:
+                this.#speaking = false;
+                this.emit(2 /* END_OF_SPEECH */, ev);
+                break;
+            }
+          }
+        };
+        const sttStreamCo = async () => {
+          for await (const ev of sttStream) {
+            if (cancelled) return;
+            if (ev.type === import_stt.SpeechEventType.FINAL_TRANSCRIPT) {
+              this.emit(3 /* FINAL_TRANSCRIPT */, ev);
+            } else if (ev.type == import_stt.SpeechEventType.INTERIM_TRANSCRIPT) {
+              this.emit(4 /* INTERIM_TRANSCRIPT */, ev);
+            }
+          }
+        };
+        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);
+        sttStream.close();
+        vadStream.close();
+        resolve();
+      });
+    }
+  }
+  get speaking() {
+    return this.#speaking;
+  }
+  get speakingProbability() {
+    return this.#speechProbability;
+  }
+  async close() {
+    if (this.#closed) {
+      throw new Error("HumanInput already closed");
+    }
+    this.#closed = true;
+    this.#room.removeAllListeners();
+    this.#speaking = false;
+    if (this.#recognizeTask) {
+      await (0, import_utils.gracefullyCancel)(this.#recognizeTask);
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  HumanInput,
+  HumanInputEvent
+});
+//# sourceMappingURL=human_input.cjs.map