@livekit/agents 0.6.4 → 0.7.1

package/dist/pipeline/agent_playout.cjs

@@ -47,14 +47,16 @@ class PlayoutHandle {
   #audioSource;
   playoutSource;
   totalPlayedTime;
+  synchronizer;
   #interrupted = false;
   pushedDuration = 0;
   intFut = new import_utils.Future();
   doneFut = new import_utils.Future();
-  constructor(speechId, audioSource, playoutSource) {
+  constructor(speechId, audioSource, playoutSource, synchronizer) {
     this.#speechId = speechId;
     this.#audioSource = audioSource;
     this.playoutSource = playoutSource;
+    this.synchronizer = synchronizer;
   }
   get speechId() {
     return this.#speechId;
@@ -95,11 +97,11 @@ class AgentPlayout extends import_node_events.default {
   set targetVolume(vol) {
     this.#targetVolume = vol;
   }
-  play(speechId, playoutSource) {
+  play(speechId, playoutSource, synchronizer) {
     if (this.#closed) {
       throw new Error("source closed");
     }
-    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);
     this.#playoutTask = this.#playout(handle, this.#playoutTask);
     return handle;
   }
@@ -109,6 +111,7 @@ class AgentPlayout extends import_node_events.default {
         captureTask.cancel();
         handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
         if (handle.interrupted || captureTask.error) {
+          handle.synchronizer.close(true);
           this.#audioSource.clearQueue();
         }
         if (!firstFrame) {
@@ -139,12 +142,15 @@ class AgentPlayout extends import_node_events.default {
           if (firstFrame) {
             this.#logger.child({ speechId: handle.speechId }).debug("started playing the first time");
             this.emit(0 /* PLAYOUT_STARTED */);
+            handle.synchronizer.segmentPlayoutStarted();
             firstFrame = false;
           }
           handle.pushedDuration += frame.samplesPerChannel / frame.sampleRate * 1e3;
+          handle.synchronizer.pushAudio(frame);
           await this.#audioSource.captureFrame(frame);
           await this.#audioSource.waitForPlayout();
         }
+        handle.synchronizer.close(false);
         resolve2();
       });
       try {

package/dist/pipeline/agent_playout.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, AudioSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport { log } from '../log.js';\nimport { CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { SynthesisHandle } from './agent_output.js';\n\nexport enum AgentPlayoutEvent {\n  PLAYOUT_STARTED,\n  PLAYOUT_STOPPED,\n}\n\nexport type AgentPlayoutCallbacks = {\n  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;\n  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;\n};\n\nexport class PlayoutHandle {\n  #speechId: string;\n  #audioSource: AudioSource;\n  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;\n  totalPlayedTime?: number;\n  #interrupted = false;\n  pushedDuration = 0;\n  intFut = new Future();\n  doneFut = new Future();\n\n  constructor(\n    speechId: string,\n    audioSource: AudioSource,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ) {\n    this.#speechId = speechId;\n    this.#audioSource = audioSource;\n    this.playoutSource = playoutSource;\n  }\n\n  get speechId(): string {\n    return this.#speechId;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get timePlayed(): number {\n    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.done) {\n      return;\n    }\n\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n\n  join(): Future {\n    return this.doneFut;\n  }\n}\n\nexport class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {\n  #closed = false;\n  #audioSource: AudioSource;\n  #targetVolume = 1;\n  #playoutTask?: CancellablePromise<void>;\n  #logger = log();\n\n  constructor(audioSource: AudioSource) {\n    super();\n    this.#audioSource = audioSource;\n  }\n\n  get targetVolume(): number {\n    return this.#targetVolume;\n  }\n\n  set targetVolume(vol: number) {\n    this.#targetVolume = vol;\n  }\n\n  play(\n    speechId: string,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ): PlayoutHandle {\n    if (this.#closed) {\n      throw new Error('source closed');\n    }\n\n    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);\n\n    this.#playoutTask = this.#playout(handle, this.#playoutTask);\n    return handle;\n  }\n\n  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      const cancel = () => {\n        captureTask.cancel();\n        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n        if (handle.interrupted || captureTask.error) {\n          this.#audioSource.clearQueue(); // make sure to remove any queued frames\n        }\n\n        if (!firstFrame) {\n          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);\n        }\n\n        handle.doneFut.resolve();\n\n        this.#logger\n          .child({ speechId: handle.speechId, interrupted: handle.interrupted })\n          .debug('playout finished');\n      };\n\n      onCancel(() => {\n        cancel();\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      if (this.#audioSource.queuedDuration > 0) {\n        // this should not happen, but log it just in case\n        this.#logger\n          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })\n          .warn('new playout while the source is still playing');\n      }\n\n      let firstFrame = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        for await (const frame of handle.playoutSource) {\n          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {\n            break;\n          }\n          if (firstFrame) {\n            this.#logger\n              .child({ speechId: handle.speechId })\n              .debug('started playing the first time');\n            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);\n            firstFrame = false;\n          }\n          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;\n          await this.#audioSource.captureFrame(frame);\n          await this.#audioSource.waitForPlayout();\n        }\n\n        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,\n        //            but for some reason too many TTS frames can gunk up the buffer and lead to\n        //            FFI errors. this works 🤷‍♀️\n        // if (this.#audioSource.queuedDuration > 0) {\n        //   await this.#audioSource.waitForPlayout();\n        // }\n\n        resolve();\n      });\n\n      try {\n        await Promise.any([captureTask, handle.intFut.await]);\n      } finally {\n        cancel();\n        resolve();\n      }\n    });\n  }\n\n  async close() {\n    this.#closed = true;\n    await this.#playoutTask;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAAyB;AACzB,iBAAoB;AACpB,mBAA6D;AAC7D,0BAAgC;AAEzB,IAAK,oBAAL,kBAAKA,uBAAL;AACL,EAAAA,sCAAA;AACA,EAAAA,sCAAA;AAFU,SAAAA;AAAA,GAAA;AAUL,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,SAAS,IAAI,oBAAO;AAAA,EACpB,UAAU,IAAI,oBAAO;AAAA,EAErB,YACE,UACA,aACA,eACA;AACA,SAAK,YAAY;AACjB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK,mBAAmB,KAAK,iBAAiB,KAAK,aAAa;AAAA,EACzE;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,MAAM;AACb;AAAA,IACF;AAEA,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,OAAe;AACb,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,qBAAsB,mBAAAC,QAA+D;AAAA,EAChG,UAAU;AAAA,EACV;AAAA,EACA,gBAAgB;AAAA,EAChB;AAAA,EACA,cAAU,gBAAI;AAAA,EAEd,YAAY,aAA0B;AACpC,UAAM;AACN,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAa,KAAa;AAC5B,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,UACA,eACe;AACf,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,eAAe;AAAA,IACjC;AAEA,UAAM,SAAS,IAAI,cAAc,UAAU,KAAK,cAAc,aAAa;AAE3E,SAAK,eAAe,KAAK,SAAS,QAAQ,KAAK,YAAY;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,QAAuB,SAA8D;AAC5F,WAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,YAAM,SAAS,MAAM;AACnB,oBAAY,OAAO;AACnB,eAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,YAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,eAAK,aAAa,WAAW;AAAA,QAC/B;AAEA,YAAI,CAAC,YAAY;AACf,eAAK,KAAK,yBAAmC,OAAO,WAAW;AAAA,QACjE;AAEA,eAAO,QAAQ,QAAQ;AAEvB,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,aAAa,OAAO,YAAY,CAAC,EACpE,MAAM,kBAAkB;AAAA,MAC7B;AAEA,eAAS,MAAM;AACb,eAAO;AAAA,MACT,CAAC;AAED,UAAI,SAAS;AACX,kBAAM,+BAAiB,OAAO;AAAA,MAChC;AAEA,UAAI,KAAK,aAAa,iBAAiB,GAAG;AAExC,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,gBAAgB,KAAK,aAAa,eAAe,CAAC,EACrF,KAAK,+CAA+C;AAAA,MACzD;AAEA,UAAI,aAAa;AAGjB,YAAM,cAAc,IAAI,gCAAyB,OAAOC,UAASC,IAAGC,cAAa;AAC/E,YAAI,YAAY;AAChB,QAAAA,UAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,yBAAiB,SAAS,OAAO,eAAe;AAC9C,cAAI,aAAa,UAAU,oCAAgB,gBAAgB;AACzD;AAAA,UACF;AACA,cAAI,YAAY;AACd,iBAAK,QACF,MAAM,EAAE,UAAU,OAAO,SAAS,CAAC,EACnC,MAAM,gCAAgC;AACzC,iBAAK,KAAK,uBAAiC;AAC3C,yBAAa;AAAA,UACf;AACA,iBAAO,kBAAmB,MAAM,oBAAoB,MAAM,aAAc;AACxE,gBAAM,KAAK,aAAa,aAAa,KAAK;AAC1C,gBAAM,KAAK,aAAa,eAAe;AAAA,QACzC;AASA,QAAAF,SAAQ;AAAA,MACV,CAAC;AAED,UAAI;AACF,cAAM,QAAQ,IAAI,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,MACtD,UAAE;AACA,eAAO;AACP,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AACF;","names":["AgentPlayoutEvent","EventEmitter","resolve","_","onCancel"]}
+{"version":3,"sources":["../../src/pipeline/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, AudioSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport { log } from '../log.js';\nimport type { TextAudioSynchronizer } from '../transcription.js';\nimport { CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { SynthesisHandle } from './agent_output.js';\n\nexport enum AgentPlayoutEvent {\n  PLAYOUT_STARTED,\n  PLAYOUT_STOPPED,\n}\n\nexport type AgentPlayoutCallbacks = {\n  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;\n  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;\n};\n\nexport class PlayoutHandle {\n  #speechId: string;\n  #audioSource: AudioSource;\n  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;\n  totalPlayedTime?: number;\n  synchronizer: TextAudioSynchronizer;\n  #interrupted = false;\n  pushedDuration = 0;\n  intFut = new Future();\n  doneFut = new Future();\n\n  constructor(\n    speechId: string,\n    audioSource: AudioSource,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n    synchronizer: TextAudioSynchronizer,\n  ) {\n    this.#speechId = speechId;\n    this.#audioSource = audioSource;\n    this.playoutSource = playoutSource;\n    this.synchronizer = synchronizer;\n  }\n\n  get speechId(): string {\n    return this.#speechId;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get timePlayed(): number {\n    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.done) {\n      return;\n    }\n\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n\n  join(): Future {\n    return this.doneFut;\n  }\n}\n\nexport class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {\n  #closed = false;\n  #audioSource: AudioSource;\n  #targetVolume = 1;\n  #playoutTask?: CancellablePromise<void>;\n  #logger = log();\n\n  constructor(audioSource: AudioSource) {\n    super();\n    this.#audioSource = audioSource;\n  }\n\n  get targetVolume(): number {\n    return this.#targetVolume;\n  }\n\n  set targetVolume(vol: number) {\n    this.#targetVolume = vol;\n  }\n\n  play(\n    speechId: string,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n    synchronizer: TextAudioSynchronizer,\n  ): PlayoutHandle {\n    if (this.#closed) {\n      throw new Error('source closed');\n    }\n\n    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);\n\n    this.#playoutTask = this.#playout(handle, this.#playoutTask);\n    return handle;\n  }\n\n  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      const cancel = () => {\n        captureTask.cancel();\n        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n        if (handle.interrupted || captureTask.error) {\n          handle.synchronizer.close(true);\n          this.#audioSource.clearQueue(); // make sure to remove any queued frames\n        }\n\n        if (!firstFrame) {\n          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);\n        }\n\n        handle.doneFut.resolve();\n\n        this.#logger\n          .child({ speechId: handle.speechId, interrupted: handle.interrupted })\n          .debug('playout finished');\n      };\n\n      onCancel(() => {\n        cancel();\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      if (this.#audioSource.queuedDuration > 0) {\n        // this should not happen, but log it just in case\n        this.#logger\n          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })\n          .warn('new playout while the source is still playing');\n      }\n\n      let firstFrame = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        for await (const frame of handle.playoutSource) {\n          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {\n            break;\n          }\n          if (firstFrame) {\n            this.#logger\n              .child({ speechId: handle.speechId })\n              .debug('started playing the first time');\n            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);\n            handle.synchronizer.segmentPlayoutStarted();\n            firstFrame = false;\n          }\n          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;\n          handle.synchronizer.pushAudio(frame);\n          await this.#audioSource.captureFrame(frame);\n          await this.#audioSource.waitForPlayout();\n        }\n\n        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,\n        //            but for some reason too many TTS frames can gunk up the buffer and lead to\n        //            FFI errors. this works 🤷‍♀️\n        // if (this.#audioSource.queuedDuration > 0) {\n        //   await this.#audioSource.waitForPlayout();\n        // }\n\n        handle.synchronizer.close(false);\n        resolve();\n      });\n\n      try {\n        await Promise.any([captureTask, handle.intFut.await]);\n      } finally {\n        cancel();\n        resolve();\n      }\n    });\n  }\n\n  async close() {\n    this.#closed = true;\n    await this.#playoutTask;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAAyB;AACzB,iBAAoB;AAEpB,mBAA6D;AAC7D,0BAAgC;AAEzB,IAAK,oBAAL,kBAAKA,uBAAL;AACL,EAAAA,sCAAA;AACA,EAAAA,sCAAA;AAFU,SAAAA;AAAA,GAAA;AAUL,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,SAAS,IAAI,oBAAO;AAAA,EACpB,UAAU,IAAI,oBAAO;AAAA,EAErB,YACE,UACA,aACA,eACA,cACA;AACA,SAAK,YAAY;AACjB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AACrB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK,mBAAmB,KAAK,iBAAiB,KAAK,aAAa;AAAA,EACzE;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,MAAM;AACb;AAAA,IACF;AAEA,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,OAAe;AACb,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,qBAAsB,mBAAAC,QAA+D;AAAA,EAChG,UAAU;AAAA,EACV;AAAA,EACA,gBAAgB;AAAA,EAChB;AAAA,EACA,cAAU,gBAAI;AAAA,EAEd,YAAY,aAA0B;AACpC,UAAM;AACN,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAa,KAAa;AAC5B,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,UACA,eACA,cACe;AACf,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,eAAe;AAAA,IACjC;AAEA,UAAM,SAAS,IAAI,cAAc,UAAU,KAAK,cAAc,eAAe,YAAY;AAEzF,SAAK,eAAe,KAAK,SAAS,QAAQ,KAAK,YAAY;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,QAAuB,SAA8D;AAC5F,WAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,YAAM,SAAS,MAAM;AACnB,oBAAY,OAAO;AACnB,eAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,YAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,iBAAO,aAAa,MAAM,IAAI;AAC9B,eAAK,aAAa,WAAW;AAAA,QAC/B;AAEA,YAAI,CAAC,YAAY;AACf,eAAK,KAAK,yBAAmC,OAAO,WAAW;AAAA,QACjE;AAEA,eAAO,QAAQ,QAAQ;AAEvB,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,aAAa,OAAO,YAAY,CAAC,EACpE,MAAM,kBAAkB;AAAA,MAC7B;AAEA,eAAS,MAAM;AACb,eAAO;AAAA,MACT,CAAC;AAED,UAAI,SAAS;AACX,kBAAM,+BAAiB,OAAO;AAAA,MAChC;AAEA,UAAI,KAAK,aAAa,iBAAiB,GAAG;AAExC,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,gBAAgB,KAAK,aAAa,eAAe,CAAC,EACrF,KAAK,+CAA+C;AAAA,MACzD;AAEA,UAAI,aAAa;AAGjB,YAAM,cAAc,IAAI,gCAAyB,OAAOC,UAASC,IAAGC,cAAa;AAC/E,YAAI,YAAY;AAChB,QAAAA,UAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,yBAAiB,SAAS,OAAO,eAAe;AAC9C,cAAI,aAAa,UAAU,oCAAgB,gBAAgB;AACzD;AAAA,UACF;AACA,cAAI,YAAY;AACd,iBAAK,QACF,MAAM,EAAE,UAAU,OAAO,SAAS,CAAC,EACnC,MAAM,gCAAgC;AACzC,iBAAK,KAAK,uBAAiC;AAC3C,mBAAO,aAAa,sBAAsB;AAC1C,yBAAa;AAAA,UACf;AACA,iBAAO,kBAAmB,MAAM,oBAAoB,MAAM,aAAc;AACxE,iBAAO,aAAa,UAAU,KAAK;AACnC,gBAAM,KAAK,aAAa,aAAa,KAAK;AAC1C,gBAAM,KAAK,aAAa,eAAe;AAAA,QACzC;AASA,eAAO,aAAa,MAAM,KAAK;AAC/B,QAAAF,SAAQ;AAAA,MACV,CAAC;AAED,UAAI;AACF,cAAM,QAAQ,IAAI,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,MACtD,UAAE;AACA,eAAO;AACP,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AACF;","names":["AgentPlayoutEvent","EventEmitter","resolve","_","onCancel"]}

package/dist/pipeline/agent_playout.d.ts

@@ -1,5 +1,6 @@
 import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { Future } from '../utils.js';
 import { SynthesisHandle } from './agent_output.js';
 export declare enum AgentPlayoutEvent {
@@ -14,10 +15,11 @@ export declare class PlayoutHandle {
     #private;
     playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
     totalPlayedTime?: number;
+    synchronizer: TextAudioSynchronizer;
     pushedDuration: number;
     intFut: Future;
     doneFut: Future;
-    constructor(speechId: string, audioSource: AudioSource, playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>);
+    constructor(speechId: string, audioSource: AudioSource, playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>, synchronizer: TextAudioSynchronizer);
     get speechId(): string;
     get interrupted(): boolean;
     get timePlayed(): number;
@@ -31,7 +33,7 @@ export declare class AgentPlayout extends AgentPlayout_base {
     constructor(audioSource: AudioSource);
     get targetVolume(): number;
     set targetVolume(vol: number);
-    play(speechId: string, playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>): PlayoutHandle;
+    play(speechId: string, playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>, synchronizer: TextAudioSynchronizer): PlayoutHandle;
     close(): Promise<void>;
 }
 export {};

package/dist/pipeline/agent_playout.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_playout.d.ts","sourceRoot":"","sources":["../../src/pipeline/agent_playout.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,EAAsB,MAAM,EAAoB,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,oBAAY,iBAAiB;IAC3B,eAAe,IAAA;IACf,eAAe,IAAA;CAChB;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,iBAAiB,CAAC,eAAe,CAAC,EAAE,MAAM,IAAI,CAAC;IAChD,CAAC,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,KAAK,IAAI,CAAC;CACnE,CAAC;AAEF,qBAAa,aAAa;;IAGxB,aAAa,EAAE,aAAa,CAAC,UAAU,GAAG,OAAO,eAAe,CAAC,cAAc,CAAC,CAAC;IACjF,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,cAAc,SAAK;IACnB,MAAM,SAAgB;IACtB,OAAO,SAAgB;gBAGrB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,EACxB,aAAa,EAAE,aAAa,CAAC,UAAU,GAAG,OAAO,eAAe,CAAC,cAAc,CAAC;IAOlF,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,SAAS;IAST,IAAI,IAAI,MAAM;CAGf;2CAE4D,aAAa,qBAAqB,CAAC;AAAhG,qBAAa,YAAa,SAAQ,iBAA+D;;gBAOnF,WAAW,EAAE,WAAW;IAKpC,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,YAAY,CAAC,GAAG,EAAE,MAAM,EAE3B;IAED,IAAI,CACF,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,aAAa,CAAC,UAAU,GAAG,OAAO,eAAe,CAAC,cAAc,CAAC,GAC/E,aAAa;IA2FV,KAAK;CAIZ"}
+{"version":3,"file":"agent_playout.d.ts","sourceRoot":"","sources":["../../src/pipeline/agent_playout.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAsB,MAAM,EAAoB,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,oBAAY,iBAAiB;IAC3B,eAAe,IAAA;IACf,eAAe,IAAA;CAChB;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,iBAAiB,CAAC,eAAe,CAAC,EAAE,MAAM,IAAI,CAAC;IAChD,CAAC,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,KAAK,IAAI,CAAC;CACnE,CAAC;AAEF,qBAAa,aAAa;;IAGxB,aAAa,EAAE,aAAa,CAAC,UAAU,GAAG,OAAO,eAAe,CAAC,cAAc,CAAC,CAAC;IACjF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,qBAAqB,CAAC;IAEpC,cAAc,SAAK;IACnB,MAAM,SAAgB;IACtB,OAAO,SAAgB;gBAGrB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,EACxB,aAAa,EAAE,aAAa,CAAC,UAAU,GAAG,OAAO,eAAe,CAAC,cAAc,CAAC,EAChF,YAAY,EAAE,qBAAqB;IAQrC,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,SAAS;IAST,IAAI,IAAI,MAAM;CAGf;2CAE4D,aAAa,qBAAqB,CAAC;AAAhG,qBAAa,YAAa,SAAQ,iBAA+D;;gBAOnF,WAAW,EAAE,WAAW;IAKpC,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,YAAY,CAAC,GAAG,EAAE,MAAM,EAE3B;IAED,IAAI,CACF,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,aAAa,CAAC,UAAU,GAAG,OAAO,eAAe,CAAC,cAAc,CAAC,EAChF,YAAY,EAAE,qBAAqB,GAClC,aAAa;IA+FV,KAAK;CAIZ"}

package/dist/pipeline/agent_playout.js

@@ -12,14 +12,16 @@ class PlayoutHandle {
   #audioSource;
   playoutSource;
   totalPlayedTime;
+  synchronizer;
   #interrupted = false;
   pushedDuration = 0;
   intFut = new Future();
   doneFut = new Future();
-  constructor(speechId, audioSource, playoutSource) {
+  constructor(speechId, audioSource, playoutSource, synchronizer) {
     this.#speechId = speechId;
     this.#audioSource = audioSource;
     this.playoutSource = playoutSource;
+    this.synchronizer = synchronizer;
   }
   get speechId() {
     return this.#speechId;
@@ -60,11 +62,11 @@ class AgentPlayout extends EventEmitter {
   set targetVolume(vol) {
     this.#targetVolume = vol;
   }
-  play(speechId, playoutSource) {
+  play(speechId, playoutSource, synchronizer) {
     if (this.#closed) {
       throw new Error("source closed");
     }
-    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);
     this.#playoutTask = this.#playout(handle, this.#playoutTask);
     return handle;
   }
@@ -74,6 +76,7 @@ class AgentPlayout extends EventEmitter {
         captureTask.cancel();
         handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
         if (handle.interrupted || captureTask.error) {
+          handle.synchronizer.close(true);
           this.#audioSource.clearQueue();
         }
         if (!firstFrame) {
@@ -104,12 +107,15 @@ class AgentPlayout extends EventEmitter {
           if (firstFrame) {
             this.#logger.child({ speechId: handle.speechId }).debug("started playing the first time");
             this.emit(0 /* PLAYOUT_STARTED */);
+            handle.synchronizer.segmentPlayoutStarted();
             firstFrame = false;
           }
           handle.pushedDuration += frame.samplesPerChannel / frame.sampleRate * 1e3;
+          handle.synchronizer.pushAudio(frame);
           await this.#audioSource.captureFrame(frame);
           await this.#audioSource.waitForPlayout();
         }
+        handle.synchronizer.close(false);
         resolve2();
       });
       try {

package/dist/pipeline/agent_playout.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, AudioSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport { log } from '../log.js';\nimport { CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { SynthesisHandle } from './agent_output.js';\n\nexport enum AgentPlayoutEvent {\n  PLAYOUT_STARTED,\n  PLAYOUT_STOPPED,\n}\n\nexport type AgentPlayoutCallbacks = {\n  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;\n  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;\n};\n\nexport class PlayoutHandle {\n  #speechId: string;\n  #audioSource: AudioSource;\n  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;\n  totalPlayedTime?: number;\n  #interrupted = false;\n  pushedDuration = 0;\n  intFut = new Future();\n  doneFut = new Future();\n\n  constructor(\n    speechId: string,\n    audioSource: AudioSource,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ) {\n    this.#speechId = speechId;\n    this.#audioSource = audioSource;\n    this.playoutSource = playoutSource;\n  }\n\n  get speechId(): string {\n    return this.#speechId;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get timePlayed(): number {\n    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.done) {\n      return;\n    }\n\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n\n  join(): Future {\n    return this.doneFut;\n  }\n}\n\nexport class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {\n  #closed = false;\n  #audioSource: AudioSource;\n  #targetVolume = 1;\n  #playoutTask?: CancellablePromise<void>;\n  #logger = log();\n\n  constructor(audioSource: AudioSource) {\n    super();\n    this.#audioSource = audioSource;\n  }\n\n  get targetVolume(): number {\n    return this.#targetVolume;\n  }\n\n  set targetVolume(vol: number) {\n    this.#targetVolume = vol;\n  }\n\n  play(\n    speechId: string,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n  ): PlayoutHandle {\n    if (this.#closed) {\n      throw new Error('source closed');\n    }\n\n    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);\n\n    this.#playoutTask = this.#playout(handle, this.#playoutTask);\n    return handle;\n  }\n\n  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      const cancel = () => {\n        captureTask.cancel();\n        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n        if (handle.interrupted || captureTask.error) {\n          this.#audioSource.clearQueue(); // make sure to remove any queued frames\n        }\n\n        if (!firstFrame) {\n          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);\n        }\n\n        handle.doneFut.resolve();\n\n        this.#logger\n          .child({ speechId: handle.speechId, interrupted: handle.interrupted })\n          .debug('playout finished');\n      };\n\n      onCancel(() => {\n        cancel();\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      if (this.#audioSource.queuedDuration > 0) {\n        // this should not happen, but log it just in case\n        this.#logger\n          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })\n          .warn('new playout while the source is still playing');\n      }\n\n      let firstFrame = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        for await (const frame of handle.playoutSource) {\n          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {\n            break;\n          }\n          if (firstFrame) {\n            this.#logger\n              .child({ speechId: handle.speechId })\n              .debug('started playing the first time');\n            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);\n            firstFrame = false;\n          }\n          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;\n          await this.#audioSource.captureFrame(frame);\n          await this.#audioSource.waitForPlayout();\n        }\n\n        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,\n        //            but for some reason too many TTS frames can gunk up the buffer and lead to\n        //            FFI errors. this works 🤷‍♀️\n        // if (this.#audioSource.queuedDuration > 0) {\n        //   await this.#audioSource.waitForPlayout();\n        // }\n\n        resolve();\n      });\n\n      try {\n        await Promise.any([captureTask, handle.intFut.await]);\n      } finally {\n        cancel();\n        resolve();\n      }\n    });\n  }\n\n  async close() {\n    this.#closed = true;\n    await this.#playoutTask;\n  }\n}\n"],"mappings":"AAKA,OAAO,kBAAkB;AACzB,SAAS,WAAW;AACpB,SAAS,oBAAoB,QAAQ,wBAAwB;AAC7D,SAAS,uBAAuB;AAEzB,IAAK,oBAAL,kBAAKA,uBAAL;AACL,EAAAA,sCAAA;AACA,EAAAA,sCAAA;AAFU,SAAAA;AAAA,GAAA;AAUL,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,SAAS,IAAI,OAAO;AAAA,EACpB,UAAU,IAAI,OAAO;AAAA,EAErB,YACE,UACA,aACA,eACA;AACA,SAAK,YAAY;AACjB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK,mBAAmB,KAAK,iBAAiB,KAAK,aAAa;AAAA,EACzE;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,MAAM;AACb;AAAA,IACF;AAEA,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,OAAe;AACb,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,qBAAsB,aAA+D;AAAA,EAChG,UAAU;AAAA,EACV;AAAA,EACA,gBAAgB;AAAA,EAChB;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,YAAY,aAA0B;AACpC,UAAM;AACN,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAa,KAAa;AAC5B,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,UACA,eACe;AACf,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,eAAe;AAAA,IACjC;AAEA,UAAM,SAAS,IAAI,cAAc,UAAU,KAAK,cAAc,aAAa;AAE3E,SAAK,eAAe,KAAK,SAAS,QAAQ,KAAK,YAAY;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,QAAuB,SAA8D;AAC5F,WAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,YAAM,SAAS,MAAM;AACnB,oBAAY,OAAO;AACnB,eAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,YAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,eAAK,aAAa,WAAW;AAAA,QAC/B;AAEA,YAAI,CAAC,YAAY;AACf,eAAK,KAAK,yBAAmC,OAAO,WAAW;AAAA,QACjE;AAEA,eAAO,QAAQ,QAAQ;AAEvB,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,aAAa,OAAO,YAAY,CAAC,EACpE,MAAM,kBAAkB;AAAA,MAC7B;AAEA,eAAS,MAAM;AACb,eAAO;AAAA,MACT,CAAC;AAED,UAAI,SAAS;AACX,cAAM,iBAAiB,OAAO;AAAA,MAChC;AAEA,UAAI,KAAK,aAAa,iBAAiB,GAAG;AAExC,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,gBAAgB,KAAK,aAAa,eAAe,CAAC,EACrF,KAAK,+CAA+C;AAAA,MACzD;AAEA,UAAI,aAAa;AAGjB,YAAM,cAAc,IAAI,mBAAyB,OAAOC,UAASC,IAAGC,cAAa;AAC/E,YAAI,YAAY;AAChB,QAAAA,UAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,yBAAiB,SAAS,OAAO,eAAe;AAC9C,cAAI,aAAa,UAAU,gBAAgB,gBAAgB;AACzD;AAAA,UACF;AACA,cAAI,YAAY;AACd,iBAAK,QACF,MAAM,EAAE,UAAU,OAAO,SAAS,CAAC,EACnC,MAAM,gCAAgC;AACzC,iBAAK,KAAK,uBAAiC;AAC3C,yBAAa;AAAA,UACf;AACA,iBAAO,kBAAmB,MAAM,oBAAoB,MAAM,aAAc;AACxE,gBAAM,KAAK,aAAa,aAAa,KAAK;AAC1C,gBAAM,KAAK,aAAa,eAAe;AAAA,QACzC;AASA,QAAAF,SAAQ;AAAA,MACV,CAAC;AAED,UAAI;AACF,cAAM,QAAQ,IAAI,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,MACtD,UAAE;AACA,eAAO;AACP,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AACF;","names":["AgentPlayoutEvent","resolve","_","onCancel"]}
+{"version":3,"sources":["../../src/pipeline/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, AudioSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport { log } from '../log.js';\nimport type { TextAudioSynchronizer } from '../transcription.js';\nimport { CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { SynthesisHandle } from './agent_output.js';\n\nexport enum AgentPlayoutEvent {\n  PLAYOUT_STARTED,\n  PLAYOUT_STOPPED,\n}\n\nexport type AgentPlayoutCallbacks = {\n  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;\n  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;\n};\n\nexport class PlayoutHandle {\n  #speechId: string;\n  #audioSource: AudioSource;\n  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;\n  totalPlayedTime?: number;\n  synchronizer: TextAudioSynchronizer;\n  #interrupted = false;\n  pushedDuration = 0;\n  intFut = new Future();\n  doneFut = new Future();\n\n  constructor(\n    speechId: string,\n    audioSource: AudioSource,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n    synchronizer: TextAudioSynchronizer,\n  ) {\n    this.#speechId = speechId;\n    this.#audioSource = audioSource;\n    this.playoutSource = playoutSource;\n    this.synchronizer = synchronizer;\n  }\n\n  get speechId(): string {\n    return this.#speechId;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get timePlayed(): number {\n    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.done) {\n      return;\n    }\n\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n\n  join(): Future {\n    return this.doneFut;\n  }\n}\n\nexport class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {\n  #closed = false;\n  #audioSource: AudioSource;\n  #targetVolume = 1;\n  #playoutTask?: CancellablePromise<void>;\n  #logger = log();\n\n  constructor(audioSource: AudioSource) {\n    super();\n    this.#audioSource = audioSource;\n  }\n\n  get targetVolume(): number {\n    return this.#targetVolume;\n  }\n\n  set targetVolume(vol: number) {\n    this.#targetVolume = vol;\n  }\n\n  play(\n    speechId: string,\n    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,\n    synchronizer: TextAudioSynchronizer,\n  ): PlayoutHandle {\n    if (this.#closed) {\n      throw new Error('source closed');\n    }\n\n    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);\n\n    this.#playoutTask = this.#playout(handle, this.#playoutTask);\n    return handle;\n  }\n\n  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      const cancel = () => {\n        captureTask.cancel();\n        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n        if (handle.interrupted || captureTask.error) {\n          handle.synchronizer.close(true);\n          this.#audioSource.clearQueue(); // make sure to remove any queued frames\n        }\n\n        if (!firstFrame) {\n          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);\n        }\n\n        handle.doneFut.resolve();\n\n        this.#logger\n          .child({ speechId: handle.speechId, interrupted: handle.interrupted })\n          .debug('playout finished');\n      };\n\n      onCancel(() => {\n        cancel();\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      if (this.#audioSource.queuedDuration > 0) {\n        // this should not happen, but log it just in case\n        this.#logger\n          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })\n          .warn('new playout while the source is still playing');\n      }\n\n      let firstFrame = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        for await (const frame of handle.playoutSource) {\n          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {\n            break;\n          }\n          if (firstFrame) {\n            this.#logger\n              .child({ speechId: handle.speechId })\n              .debug('started playing the first time');\n            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);\n            handle.synchronizer.segmentPlayoutStarted();\n            firstFrame = false;\n          }\n          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;\n          handle.synchronizer.pushAudio(frame);\n          await this.#audioSource.captureFrame(frame);\n          await this.#audioSource.waitForPlayout();\n        }\n\n        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,\n        //            but for some reason too many TTS frames can gunk up the buffer and lead to\n        //            FFI errors. this works 🤷‍♀️\n        // if (this.#audioSource.queuedDuration > 0) {\n        //   await this.#audioSource.waitForPlayout();\n        // }\n\n        handle.synchronizer.close(false);\n        resolve();\n      });\n\n      try {\n        await Promise.any([captureTask, handle.intFut.await]);\n      } finally {\n        cancel();\n        resolve();\n      }\n    });\n  }\n\n  async close() {\n    this.#closed = true;\n    await this.#playoutTask;\n  }\n}\n"],"mappings":"AAKA,OAAO,kBAAkB;AACzB,SAAS,WAAW;AAEpB,SAAS,oBAAoB,QAAQ,wBAAwB;AAC7D,SAAS,uBAAuB;AAEzB,IAAK,oBAAL,kBAAKA,uBAAL;AACL,EAAAA,sCAAA;AACA,EAAAA,sCAAA;AAFU,SAAAA;AAAA,GAAA;AAUL,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,SAAS,IAAI,OAAO;AAAA,EACpB,UAAU,IAAI,OAAO;AAAA,EAErB,YACE,UACA,aACA,eACA,cACA;AACA,SAAK,YAAY;AACjB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AACrB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK,mBAAmB,KAAK,iBAAiB,KAAK,aAAa;AAAA,EACzE;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,MAAM;AACb;AAAA,IACF;AAEA,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,OAAe;AACb,WAAO,KAAK;AAAA,EACd;AACF;AAEO,MAAM,qBAAsB,aAA+D;AAAA,EAChG,UAAU;AAAA,EACV;AAAA,EACA,gBAAgB;AAAA,EAChB;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,YAAY,aAA0B;AACpC,UAAM;AACN,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,aAAa,KAAa;AAC5B,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,UACA,eACA,cACe;AACf,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,eAAe;AAAA,IACjC;AAEA,UAAM,SAAS,IAAI,cAAc,UAAU,KAAK,cAAc,eAAe,YAAY;AAEzF,SAAK,eAAe,KAAK,SAAS,QAAQ,KAAK,YAAY;AAC3D,WAAO;AAAA,EACT;AAAA,EAEA,SAAS,QAAuB,SAA8D;AAC5F,WAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,YAAM,SAAS,MAAM;AACnB,oBAAY,OAAO;AACnB,eAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,YAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,iBAAO,aAAa,MAAM,IAAI;AAC9B,eAAK,aAAa,WAAW;AAAA,QAC/B;AAEA,YAAI,CAAC,YAAY;AACf,eAAK,KAAK,yBAAmC,OAAO,WAAW;AAAA,QACjE;AAEA,eAAO,QAAQ,QAAQ;AAEvB,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,aAAa,OAAO,YAAY,CAAC,EACpE,MAAM,kBAAkB;AAAA,MAC7B;AAEA,eAAS,MAAM;AACb,eAAO;AAAA,MACT,CAAC;AAED,UAAI,SAAS;AACX,cAAM,iBAAiB,OAAO;AAAA,MAChC;AAEA,UAAI,KAAK,aAAa,iBAAiB,GAAG;AAExC,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,UAAU,gBAAgB,KAAK,aAAa,eAAe,CAAC,EACrF,KAAK,+CAA+C;AAAA,MACzD;AAEA,UAAI,aAAa;AAGjB,YAAM,cAAc,IAAI,mBAAyB,OAAOC,UAASC,IAAGC,cAAa;AAC/E,YAAI,YAAY;AAChB,QAAAA,UAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,yBAAiB,SAAS,OAAO,eAAe;AAC9C,cAAI,aAAa,UAAU,gBAAgB,gBAAgB;AACzD;AAAA,UACF;AACA,cAAI,YAAY;AACd,iBAAK,QACF,MAAM,EAAE,UAAU,OAAO,SAAS,CAAC,EACnC,MAAM,gCAAgC;AACzC,iBAAK,KAAK,uBAAiC;AAC3C,mBAAO,aAAa,sBAAsB;AAC1C,yBAAa;AAAA,UACf;AACA,iBAAO,kBAAmB,MAAM,oBAAoB,MAAM,aAAc;AACxE,iBAAO,aAAa,UAAU,KAAK;AACnC,gBAAM,KAAK,aAAa,aAAa,KAAK;AAC1C,gBAAM,KAAK,aAAa,eAAe;AAAA,QACzC;AASA,eAAO,aAAa,MAAM,KAAK;AAC/B,QAAAF,SAAQ;AAAA,MACV,CAAC;AAED,UAAI;AACF,cAAM,QAAQ,IAAI,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,MACtD,UAAE;AACA,eAAO;AACP,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AACF;","names":["AgentPlayoutEvent","resolve","_","onCancel"]}

package/dist/pipeline/human_input.cjs

@@ -57,6 +57,12 @@ class HumanInput extends import_node_events.EventEmitter {
     this.#room.on(import_rtc_node.RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));
     this.#subscribeToMicrophone();
   }
+  get participant() {
+    return this.#participant;
+  }
+  get subscribedTrack() {
+    return this.#subscribedTrack;
+  }
   #subscribeToMicrophone() {
     if (!this.#participant) {
       this.#logger.error("Participant is not set");

package/dist/pipeline/human_input.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/human_input.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { log } from '../log.js';\nimport type { STT, SpeechEvent } from '../stt/stt.js';\nimport { SpeechEventType } from '../stt/stt.js';\nimport { CancellablePromise, gracefullyCancel } from '../utils.js';\nimport type { VAD, VADEvent } from '../vad.js';\nimport { VADEventType } from '../vad.js';\n\nexport enum HumanInputEvent {\n  START_OF_SPEECH,\n  VAD_INFERENCE_DONE,\n  END_OF_SPEECH,\n  FINAL_TRANSCRIPT,\n  INTERIM_TRANSCRIPT,\n}\n\nexport type HumanInputCallbacks = {\n  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;\n  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;\n  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;\n};\n\nexport class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {\n  #closed = false;\n  #room: Room;\n  #vad: VAD;\n  #stt: STT;\n  #participant: RemoteParticipant;\n  #subscribedTrack?: RemoteAudioTrack;\n  #recognizeTask?: CancellablePromise<void>;\n  #speaking = false;\n  #speechProbability = 0;\n  #logger = log();\n\n  constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant) {\n    super();\n    this.#room = room;\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#participant = participant;\n\n    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));\n    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));\n    this.#subscribeToMicrophone();\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.#participant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.#participant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n\n    const track = microphonePublication.track;\n    if (track && track !== this.#subscribedTrack) {\n      this.#subscribedTrack = track;\n      if (this.#recognizeTask) {\n        this.#recognizeTask.cancel();\n      }\n\n      const audioStream = new AudioStream(track, 16000);\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        const sttStream = this.#stt.stream();\n        const vadStream = this.#vad.stream();\n\n        const audioStreamCo = async () => {\n          for await (const ev of audioStream) {\n            if (cancelled) return;\n            sttStream.pushFrame(ev);\n            vadStream.pushFrame(ev);\n          }\n        };\n\n        const vadStreamCo = async () => {\n          for await (const ev of vadStream) {\n            if (cancelled) return;\n            switch (ev.type) {\n              case VADEventType.START_OF_SPEECH:\n                this.#speaking = true;\n                this.emit(HumanInputEvent.START_OF_SPEECH, ev);\n                break;\n              case VADEventType.INFERENCE_DONE:\n                this.#speechProbability = ev.probability;\n                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);\n                break;\n              case VADEventType.END_OF_SPEECH:\n                this.#speaking = false;\n                this.emit(HumanInputEvent.END_OF_SPEECH, ev);\n                break;\n            }\n          }\n        };\n\n        const sttStreamCo = async () => {\n          for await (const ev of sttStream) {\n            if (cancelled) return;\n            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {\n              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);\n            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {\n              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);\n            }\n          }\n        };\n\n        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);\n        sttStream.close();\n        vadStream.close();\n        resolve();\n      });\n    }\n  }\n\n  get speaking(): boolean {\n    return this.#speaking;\n  }\n\n  get speakingProbability(): number {\n    return this.#speechProbability;\n  }\n\n  async close() {\n    if (this.#closed) {\n      throw new Error('HumanInput already closed');\n    }\n    this.#closed = true;\n    this.#room.removeAllListeners();\n    this.#speaking = false;\n    if (this.#recognizeTask) {\n      await gracefullyCancel(this.#recognizeTask);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASA,sBAAoD;AAEpD,yBAA6B;AAC7B,iBAAoB;AAEpB,iBAAgC;AAChC,mBAAqD;AAErD,iBAA6B;AAEtB,IAAK,kBAAL,kBAAKA,qBAAL;AACL,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AALU,SAAAA;AAAA,GAAA;AAgBL,MAAM,mBAAoB,gCAA6D;AAAA,EAC5F,UAAU;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,cAAU,gBAAI;AAAA,EAEd,YAAY,MAAY,KAAU,KAAU,aAAgC;AAC1E,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,MAAM,GAAG,0BAAU,gBAAgB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC9E,SAAK,MAAM,GAAG,0BAAU,iBAAiB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC/E,SAAK,uBAAuB;AAAA,EAC9B;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,aAAa,kBAAkB,OAAO,GAAG;AACtE,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAEA,UAAM,QAAQ,sBAAsB;AACpC,QAAI,SAAS,UAAU,KAAK,kBAAkB;AAC5C,WAAK,mBAAmB;AACxB,UAAI,KAAK,gBAAgB;AACvB,aAAK,eAAe,OAAO;AAAA,MAC7B;AAEA,YAAM,cAAc,IAAI,4BAAY,OAAO,IAAK;AAGhD,WAAK,iBAAiB,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC3E,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,cAAM,YAAY,KAAK,KAAK,OAAO;AACnC,cAAM,YAAY,KAAK,KAAK,OAAO;AAEnC,cAAM,gBAAgB,YAAY;AAChC,2BAAiB,MAAM,aAAa;AAClC,gBAAI,UAAW;AACf,sBAAU,UAAU,EAAE;AACtB,sBAAU,UAAU,EAAE;AAAA,UACxB;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,oBAAQ,GAAG,MAAM;AAAA,cACf,KAAK,wBAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,yBAAiC,EAAE;AAC7C;AAAA,cACF,KAAK,wBAAa;AAChB,qBAAK,qBAAqB,GAAG;AAC7B,qBAAK,KAAK,4BAAoC,EAAE;AAChD;AAAA,cACF,KAAK,wBAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,uBAA+B,EAAE;AAC3C;AAAA,YACJ;AAAA,UACF;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,gBAAI,GAAG,SAAS,2BAAgB,kBAAkB;AAChD,mBAAK,KAAK,0BAAkC,EAAE;AAAA,YAChD,WAAW,GAAG,QAAQ,2BAAgB,oBAAoB;AACxD,mBAAK,KAAK,4BAAoC,EAAE;AAAA,YAClD;AAAA,UACF;AAAA,QACF;AAEA,cAAM,QAAQ,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC;AACjE,kBAAU,MAAM;AAChB,kBAAU,MAAM;AAChB,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AACA,SAAK,UAAU;AACf,SAAK,MAAM,mBAAmB;AAC9B,SAAK,YAAY;AACjB,QAAI,KAAK,gBAAgB;AACvB,gBAAM,+BAAiB,KAAK,cAAc;AAAA,IAC5C;AAAA,EACF;AACF;","names":["HumanInputEvent"]}
+{"version":3,"sources":["../../src/pipeline/human_input.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { log } from '../log.js';\nimport type { STT, SpeechEvent } from '../stt/stt.js';\nimport { SpeechEventType } from '../stt/stt.js';\nimport { CancellablePromise, gracefullyCancel } from '../utils.js';\nimport type { VAD, VADEvent } from '../vad.js';\nimport { VADEventType } from '../vad.js';\n\nexport enum HumanInputEvent {\n  START_OF_SPEECH,\n  VAD_INFERENCE_DONE,\n  END_OF_SPEECH,\n  FINAL_TRANSCRIPT,\n  INTERIM_TRANSCRIPT,\n}\n\nexport type HumanInputCallbacks = {\n  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;\n  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;\n  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;\n};\n\nexport class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {\n  #closed = false;\n  #room: Room;\n  #vad: VAD;\n  #stt: STT;\n  #participant: RemoteParticipant;\n  #subscribedTrack?: RemoteAudioTrack;\n  #recognizeTask?: CancellablePromise<void>;\n  #speaking = false;\n  #speechProbability = 0;\n  #logger = log();\n\n  constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant) {\n    super();\n    this.#room = room;\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#participant = participant;\n\n    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));\n    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));\n    this.#subscribeToMicrophone();\n  }\n\n  get participant(): RemoteParticipant {\n    return this.#participant;\n  }\n\n  get subscribedTrack(): RemoteAudioTrack | undefined {\n    return this.#subscribedTrack;\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.#participant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.#participant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n\n    const track = microphonePublication.track;\n    if (track && track !== this.#subscribedTrack) {\n      this.#subscribedTrack = track;\n      if (this.#recognizeTask) {\n        this.#recognizeTask.cancel();\n      }\n\n      const audioStream = new AudioStream(track, 16000);\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        const sttStream = this.#stt.stream();\n        const vadStream = this.#vad.stream();\n\n        const audioStreamCo = async () => {\n          for await (const ev of audioStream) {\n            if (cancelled) return;\n            sttStream.pushFrame(ev);\n            vadStream.pushFrame(ev);\n          }\n        };\n\n        const vadStreamCo = async () => {\n          for await (const ev of vadStream) {\n            if (cancelled) return;\n            switch (ev.type) {\n              case VADEventType.START_OF_SPEECH:\n                this.#speaking = true;\n                this.emit(HumanInputEvent.START_OF_SPEECH, ev);\n                break;\n              case VADEventType.INFERENCE_DONE:\n                this.#speechProbability = ev.probability;\n                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);\n                break;\n              case VADEventType.END_OF_SPEECH:\n                this.#speaking = false;\n                this.emit(HumanInputEvent.END_OF_SPEECH, ev);\n                break;\n            }\n          }\n        };\n\n        const sttStreamCo = async () => {\n          for await (const ev of sttStream) {\n            if (cancelled) return;\n            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {\n              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);\n            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {\n              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);\n            }\n          }\n        };\n\n        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);\n        sttStream.close();\n        vadStream.close();\n        resolve();\n      });\n    }\n  }\n\n  get speaking(): boolean {\n    return this.#speaking;\n  }\n\n  get speakingProbability(): number {\n    return this.#speechProbability;\n  }\n\n  async close() {\n    if (this.#closed) {\n      throw new Error('HumanInput already closed');\n    }\n    this.#closed = true;\n    this.#room.removeAllListeners();\n    this.#speaking = false;\n    if (this.#recognizeTask) {\n      await gracefullyCancel(this.#recognizeTask);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASA,sBAAoD;AAEpD,yBAA6B;AAC7B,iBAAoB;AAEpB,iBAAgC;AAChC,mBAAqD;AAErD,iBAA6B;AAEtB,IAAK,kBAAL,kBAAKA,qBAAL;AACL,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AALU,SAAAA;AAAA,GAAA;AAgBL,MAAM,mBAAoB,gCAA6D;AAAA,EAC5F,UAAU;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,cAAU,gBAAI;AAAA,EAEd,YAAY,MAAY,KAAU,KAAU,aAAgC;AAC1E,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,MAAM,GAAG,0BAAU,gBAAgB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC9E,SAAK,MAAM,GAAG,0BAAU,iBAAiB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC/E,SAAK,uBAAuB;AAAA,EAC9B;AAAA,EAEA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAgD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,aAAa,kBAAkB,OAAO,GAAG;AACtE,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAEA,UAAM,QAAQ,sBAAsB;AACpC,QAAI,SAAS,UAAU,KAAK,kBAAkB;AAC5C,WAAK,mBAAmB;AACxB,UAAI,KAAK,gBAAgB;AACvB,aAAK,eAAe,OAAO;AAAA,MAC7B;AAEA,YAAM,cAAc,IAAI,4BAAY,OAAO,IAAK;AAGhD,WAAK,iBAAiB,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC3E,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,cAAM,YAAY,KAAK,KAAK,OAAO;AACnC,cAAM,YAAY,KAAK,KAAK,OAAO;AAEnC,cAAM,gBAAgB,YAAY;AAChC,2BAAiB,MAAM,aAAa;AAClC,gBAAI,UAAW;AACf,sBAAU,UAAU,EAAE;AACtB,sBAAU,UAAU,EAAE;AAAA,UACxB;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,oBAAQ,GAAG,MAAM;AAAA,cACf,KAAK,wBAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,yBAAiC,EAAE;AAC7C;AAAA,cACF,KAAK,wBAAa;AAChB,qBAAK,qBAAqB,GAAG;AAC7B,qBAAK,KAAK,4BAAoC,EAAE;AAChD;AAAA,cACF,KAAK,wBAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,uBAA+B,EAAE;AAC3C;AAAA,YACJ;AAAA,UACF;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,gBAAI,GAAG,SAAS,2BAAgB,kBAAkB;AAChD,mBAAK,KAAK,0BAAkC,EAAE;AAAA,YAChD,WAAW,GAAG,QAAQ,2BAAgB,oBAAoB;AACxD,mBAAK,KAAK,4BAAoC,EAAE;AAAA,YAClD;AAAA,UACF;AAAA,QACF;AAEA,cAAM,QAAQ,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC;AACjE,kBAAU,MAAM;AAChB,kBAAU,MAAM;AAChB,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AACA,SAAK,UAAU;AACf,SAAK,MAAM,mBAAmB;AAC9B,SAAK,YAAY;AACjB,QAAI,KAAK,gBAAgB;AACvB,gBAAM,+BAAiB,KAAK,cAAc;AAAA,IAC5C;AAAA,EACF;AACF;","names":["HumanInputEvent"]}

package/dist/pipeline/human_input.d.ts

@@ -1,4 +1,4 @@
-import type { RemoteParticipant, Room } from '@livekit/rtc-node';
+import type { RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { STT, SpeechEvent } from '../stt/stt.js';
 import type { VAD, VADEvent } from '../vad.js';
@@ -20,6 +20,8 @@ declare const HumanInput_base: new () => TypedEmitter<HumanInputCallbacks>;
 export declare class HumanInput extends HumanInput_base {
     #private;
     constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant);
+    get participant(): RemoteParticipant;
+    get subscribedTrack(): RemoteAudioTrack | undefined;
     get speaking(): boolean;
     get speakingProbability(): number;
     close(): Promise<void>;

package/dist/pipeline/human_input.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"human_input.d.ts","sourceRoot":"","sources":["../../src/pipeline/human_input.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,iBAAiB,EAEjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAGtD,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAG/C,oBAAY,eAAe;IACzB,eAAe,IAAA;IACf,kBAAkB,IAAA;IAClB,aAAa,IAAA;IACb,gBAAgB,IAAA;IAChB,kBAAkB,IAAA;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC7D,CAAC,eAAe,CAAC,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAChE,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC3D,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;IACjE,CAAC,eAAe,CAAC,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CACpE,CAAC;yCAEyD,aAAa,mBAAmB,CAAC;AAA5F,qBAAa,UAAW,SAAQ,eAA6D;;gBAY/E,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,iBAAiB;IAmG1E,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAEK,KAAK;CAWZ"}
+{"version":3,"file":"human_input.d.ts","sourceRoot":"","sources":["../../src/pipeline/human_input.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EAEjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAGtD,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAG/C,oBAAY,eAAe;IACzB,eAAe,IAAA;IACf,kBAAkB,IAAA;IAClB,aAAa,IAAA;IACb,gBAAgB,IAAA;IAChB,kBAAkB,IAAA;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC7D,CAAC,eAAe,CAAC,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAChE,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC3D,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;IACjE,CAAC,eAAe,CAAC,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CACpE,CAAC;yCAEyD,aAAa,mBAAmB,CAAC;AAA5F,qBAAa,UAAW,SAAQ,eAA6D;;gBAY/E,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,iBAAiB;IAY1E,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,eAAe,IAAI,gBAAgB,GAAG,SAAS,CAElD;IAyFD,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAEK,KAAK;CAWZ"}

package/dist/pipeline/human_input.js

@@ -33,6 +33,12 @@ class HumanInput extends EventEmitter {
     this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));
     this.#subscribeToMicrophone();
   }
+  get participant() {
+    return this.#participant;
+  }
+  get subscribedTrack() {
+    return this.#subscribedTrack;
+  }
   #subscribeToMicrophone() {
     if (!this.#participant) {
       this.#logger.error("Participant is not set");

package/dist/pipeline/human_input.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/human_input.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { log } from '../log.js';\nimport type { STT, SpeechEvent } from '../stt/stt.js';\nimport { SpeechEventType } from '../stt/stt.js';\nimport { CancellablePromise, gracefullyCancel } from '../utils.js';\nimport type { VAD, VADEvent } from '../vad.js';\nimport { VADEventType } from '../vad.js';\n\nexport enum HumanInputEvent {\n  START_OF_SPEECH,\n  VAD_INFERENCE_DONE,\n  END_OF_SPEECH,\n  FINAL_TRANSCRIPT,\n  INTERIM_TRANSCRIPT,\n}\n\nexport type HumanInputCallbacks = {\n  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;\n  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;\n  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;\n};\n\nexport class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {\n  #closed = false;\n  #room: Room;\n  #vad: VAD;\n  #stt: STT;\n  #participant: RemoteParticipant;\n  #subscribedTrack?: RemoteAudioTrack;\n  #recognizeTask?: CancellablePromise<void>;\n  #speaking = false;\n  #speechProbability = 0;\n  #logger = log();\n\n  constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant) {\n    super();\n    this.#room = room;\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#participant = participant;\n\n    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));\n    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));\n    this.#subscribeToMicrophone();\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.#participant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.#participant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n\n    const track = microphonePublication.track;\n    if (track && track !== this.#subscribedTrack) {\n      this.#subscribedTrack = track;\n      if (this.#recognizeTask) {\n        this.#recognizeTask.cancel();\n      }\n\n      const audioStream = new AudioStream(track, 16000);\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        const sttStream = this.#stt.stream();\n        const vadStream = this.#vad.stream();\n\n        const audioStreamCo = async () => {\n          for await (const ev of audioStream) {\n            if (cancelled) return;\n            sttStream.pushFrame(ev);\n            vadStream.pushFrame(ev);\n          }\n        };\n\n        const vadStreamCo = async () => {\n          for await (const ev of vadStream) {\n            if (cancelled) return;\n            switch (ev.type) {\n              case VADEventType.START_OF_SPEECH:\n                this.#speaking = true;\n                this.emit(HumanInputEvent.START_OF_SPEECH, ev);\n                break;\n              case VADEventType.INFERENCE_DONE:\n                this.#speechProbability = ev.probability;\n                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);\n                break;\n              case VADEventType.END_OF_SPEECH:\n                this.#speaking = false;\n                this.emit(HumanInputEvent.END_OF_SPEECH, ev);\n                break;\n            }\n          }\n        };\n\n        const sttStreamCo = async () => {\n          for await (const ev of sttStream) {\n            if (cancelled) return;\n            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {\n              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);\n            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {\n              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);\n            }\n          }\n        };\n\n        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);\n        sttStream.close();\n        vadStream.close();\n        resolve();\n      });\n    }\n  }\n\n  get speaking(): boolean {\n    return this.#speaking;\n  }\n\n  get speakingProbability(): number {\n    return this.#speechProbability;\n  }\n\n  async close() {\n    if (this.#closed) {\n      throw new Error('HumanInput already closed');\n    }\n    this.#closed = true;\n    this.#room.removeAllListeners();\n    this.#speaking = false;\n    if (this.#recognizeTask) {\n      await gracefullyCancel(this.#recognizeTask);\n    }\n  }\n}\n"],"mappings":"AASA,SAAS,aAAa,WAAW,mBAAmB;AAEpD,SAAS,oBAAoB;AAC7B,SAAS,WAAW;AAEpB,SAAS,uBAAuB;AAChC,SAAS,oBAAoB,wBAAwB;AAErD,SAAS,oBAAoB;AAEtB,IAAK,kBAAL,kBAAKA,qBAAL;AACL,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AALU,SAAAA;AAAA,GAAA;AAgBL,MAAM,mBAAoB,aAA6D;AAAA,EAC5F,UAAU;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,UAAU,IAAI;AAAA,EAEd,YAAY,MAAY,KAAU,KAAU,aAAgC;AAC1E,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,MAAM,GAAG,UAAU,gBAAgB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC9E,SAAK,MAAM,GAAG,UAAU,iBAAiB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC/E,SAAK,uBAAuB;AAAA,EAC9B;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,aAAa,kBAAkB,OAAO,GAAG;AACtE,UAAI,YAAY,WAAW,YAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAEA,UAAM,QAAQ,sBAAsB;AACpC,QAAI,SAAS,UAAU,KAAK,kBAAkB;AAC5C,WAAK,mBAAmB;AACxB,UAAI,KAAK,gBAAgB;AACvB,aAAK,eAAe,OAAO;AAAA,MAC7B;AAEA,YAAM,cAAc,IAAI,YAAY,OAAO,IAAK;AAGhD,WAAK,iBAAiB,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC3E,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,cAAM,YAAY,KAAK,KAAK,OAAO;AACnC,cAAM,YAAY,KAAK,KAAK,OAAO;AAEnC,cAAM,gBAAgB,YAAY;AAChC,2BAAiB,MAAM,aAAa;AAClC,gBAAI,UAAW;AACf,sBAAU,UAAU,EAAE;AACtB,sBAAU,UAAU,EAAE;AAAA,UACxB;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,oBAAQ,GAAG,MAAM;AAAA,cACf,KAAK,aAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,yBAAiC,EAAE;AAC7C;AAAA,cACF,KAAK,aAAa;AAChB,qBAAK,qBAAqB,GAAG;AAC7B,qBAAK,KAAK,4BAAoC,EAAE;AAChD;AAAA,cACF,KAAK,aAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,uBAA+B,EAAE;AAC3C;AAAA,YACJ;AAAA,UACF;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,gBAAI,GAAG,SAAS,gBAAgB,kBAAkB;AAChD,mBAAK,KAAK,0BAAkC,EAAE;AAAA,YAChD,WAAW,GAAG,QAAQ,gBAAgB,oBAAoB;AACxD,mBAAK,KAAK,4BAAoC,EAAE;AAAA,YAClD;AAAA,UACF;AAAA,QACF;AAEA,cAAM,QAAQ,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC;AACjE,kBAAU,MAAM;AAChB,kBAAU,MAAM;AAChB,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AACA,SAAK,UAAU;AACf,SAAK,MAAM,mBAAmB;AAC9B,SAAK,YAAY;AACjB,QAAI,KAAK,gBAAgB;AACvB,YAAM,iBAAiB,KAAK,cAAc;AAAA,IAC5C;AAAA,EACF;AACF;","names":["HumanInputEvent"]}
+{"version":3,"sources":["../../src/pipeline/human_input.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { log } from '../log.js';\nimport type { STT, SpeechEvent } from '../stt/stt.js';\nimport { SpeechEventType } from '../stt/stt.js';\nimport { CancellablePromise, gracefullyCancel } from '../utils.js';\nimport type { VAD, VADEvent } from '../vad.js';\nimport { VADEventType } from '../vad.js';\n\nexport enum HumanInputEvent {\n  START_OF_SPEECH,\n  VAD_INFERENCE_DONE,\n  END_OF_SPEECH,\n  FINAL_TRANSCRIPT,\n  INTERIM_TRANSCRIPT,\n}\n\nexport type HumanInputCallbacks = {\n  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;\n  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;\n  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;\n};\n\nexport class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {\n  #closed = false;\n  #room: Room;\n  #vad: VAD;\n  #stt: STT;\n  #participant: RemoteParticipant;\n  #subscribedTrack?: RemoteAudioTrack;\n  #recognizeTask?: CancellablePromise<void>;\n  #speaking = false;\n  #speechProbability = 0;\n  #logger = log();\n\n  constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant) {\n    super();\n    this.#room = room;\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#participant = participant;\n\n    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));\n    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));\n    this.#subscribeToMicrophone();\n  }\n\n  get participant(): RemoteParticipant {\n    return this.#participant;\n  }\n\n  get subscribedTrack(): RemoteAudioTrack | undefined {\n    return this.#subscribedTrack;\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.#participant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.#participant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n\n    const track = microphonePublication.track;\n    if (track && track !== this.#subscribedTrack) {\n      this.#subscribedTrack = track;\n      if (this.#recognizeTask) {\n        this.#recognizeTask.cancel();\n      }\n\n      const audioStream = new AudioStream(track, 16000);\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        const sttStream = this.#stt.stream();\n        const vadStream = this.#vad.stream();\n\n        const audioStreamCo = async () => {\n          for await (const ev of audioStream) {\n            if (cancelled) return;\n            sttStream.pushFrame(ev);\n            vadStream.pushFrame(ev);\n          }\n        };\n\n        const vadStreamCo = async () => {\n          for await (const ev of vadStream) {\n            if (cancelled) return;\n            switch (ev.type) {\n              case VADEventType.START_OF_SPEECH:\n                this.#speaking = true;\n                this.emit(HumanInputEvent.START_OF_SPEECH, ev);\n                break;\n              case VADEventType.INFERENCE_DONE:\n                this.#speechProbability = ev.probability;\n                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);\n                break;\n              case VADEventType.END_OF_SPEECH:\n                this.#speaking = false;\n                this.emit(HumanInputEvent.END_OF_SPEECH, ev);\n                break;\n            }\n          }\n        };\n\n        const sttStreamCo = async () => {\n          for await (const ev of sttStream) {\n            if (cancelled) return;\n            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {\n              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);\n            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {\n              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);\n            }\n          }\n        };\n\n        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);\n        sttStream.close();\n        vadStream.close();\n        resolve();\n      });\n    }\n  }\n\n  get speaking(): boolean {\n    return this.#speaking;\n  }\n\n  get speakingProbability(): number {\n    return this.#speechProbability;\n  }\n\n  async close() {\n    if (this.#closed) {\n      throw new Error('HumanInput already closed');\n    }\n    this.#closed = true;\n    this.#room.removeAllListeners();\n    this.#speaking = false;\n    if (this.#recognizeTask) {\n      await gracefullyCancel(this.#recognizeTask);\n    }\n  }\n}\n"],"mappings":"AASA,SAAS,aAAa,WAAW,mBAAmB;AAEpD,SAAS,oBAAoB;AAC7B,SAAS,WAAW;AAEpB,SAAS,uBAAuB;AAChC,SAAS,oBAAoB,wBAAwB;AAErD,SAAS,oBAAoB;AAEtB,IAAK,kBAAL,kBAAKA,qBAAL;AACL,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AALU,SAAAA;AAAA,GAAA;AAgBL,MAAM,mBAAoB,aAA6D;AAAA,EAC5F,UAAU;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,UAAU,IAAI;AAAA,EAEd,YAAY,MAAY,KAAU,KAAU,aAAgC;AAC1E,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,MAAM,GAAG,UAAU,gBAAgB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC9E,SAAK,MAAM,GAAG,UAAU,iBAAiB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC/E,SAAK,uBAAuB;AAAA,EAC9B;AAAA,EAEA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAgD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,aAAa,kBAAkB,OAAO,GAAG;AACtE,UAAI,YAAY,WAAW,YAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAEA,UAAM,QAAQ,sBAAsB;AACpC,QAAI,SAAS,UAAU,KAAK,kBAAkB;AAC5C,WAAK,mBAAmB;AACxB,UAAI,KAAK,gBAAgB;AACvB,aAAK,eAAe,OAAO;AAAA,MAC7B;AAEA,YAAM,cAAc,IAAI,YAAY,OAAO,IAAK;AAGhD,WAAK,iBAAiB,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC3E,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,cAAM,YAAY,KAAK,KAAK,OAAO;AACnC,cAAM,YAAY,KAAK,KAAK,OAAO;AAEnC,cAAM,gBAAgB,YAAY;AAChC,2BAAiB,MAAM,aAAa;AAClC,gBAAI,UAAW;AACf,sBAAU,UAAU,EAAE;AACtB,sBAAU,UAAU,EAAE;AAAA,UACxB;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,oBAAQ,GAAG,MAAM;AAAA,cACf,KAAK,aAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,yBAAiC,EAAE;AAC7C;AAAA,cACF,KAAK,aAAa;AAChB,qBAAK,qBAAqB,GAAG;AAC7B,qBAAK,KAAK,4BAAoC,EAAE;AAChD;AAAA,cACF,KAAK,aAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,uBAA+B,EAAE;AAC3C;AAAA,YACJ;AAAA,UACF;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,gBAAI,GAAG,SAAS,gBAAgB,kBAAkB;AAChD,mBAAK,KAAK,0BAAkC,EAAE;AAAA,YAChD,WAAW,GAAG,QAAQ,gBAAgB,oBAAoB;AACxD,mBAAK,KAAK,4BAAoC,EAAE;AAAA,YAClD;AAAA,UACF;AAAA,QACF;AAEA,cAAM,QAAQ,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC;AACjE,kBAAU,MAAM;AAChB,kBAAU,MAAM;AAChB,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AACA,SAAK,UAAU;AACf,SAAK,MAAM,mBAAmB;AAC9B,SAAK,YAAY;AACjB,QAAI,KAAK,gBAAgB;AACvB,YAAM,iBAAiB,KAAK,cAAc;AAAA,IAC5C;AAAA,EACF;AACF;","names":["HumanInputEvent"]}

package/dist/pipeline/pipeline_agent.cjs

@@ -35,12 +35,14 @@ __export(pipeline_agent_exports, {
 });
 module.exports = __toCommonJS(pipeline_agent_exports);
 var import_rtc_node = require("@livekit/rtc-node");
+var import_node_crypto = require("node:crypto");
 var import_node_events = __toESM(require("node:events"), 1);
 var import_llm = require("../llm/index.cjs");
 var import_llm2 = require("../llm/index.cjs");
 var import_log = require("../log.cjs");
 var import_stt = require("../stt/index.cjs");
 var import_basic = require("../tokenize/basic/index.cjs");
+var import_transcription = require("../transcription.cjs");
 var import_tts = require("../tts/index.cjs");
 var import_utils = require("../utils.cjs");
 var import_vad = require("../vad.cjs");
@@ -137,7 +139,7 @@ class VoicePipelineAgent extends import_node_events.default {
   #pendingAgentReply;
   #agentReplyTask;
   #playingSpeech;
-  #transcribedText = "";
+  transcribedText = "";
   #transcribedInterimText = "";
   #speechQueueOpen = new import_utils.Future();
   #speechQueue = new import_utils.AsyncIterableQueue();
@@ -150,6 +152,8 @@ class VoicePipelineAgent extends import_node_events.default {
   #agentPublication;
   #lastFinalTranscriptTime;
   #lastSpeechTime;
+  #transcriptionId;
+  #agentTranscribedText = "";
   constructor(vad, stt, llm, tts, opts = defaultVPAOptions) {
     super();
     this.#opts = { ...defaultVPAOptions, ...opts };
@@ -165,7 +169,9 @@ class VoicePipelineAgent extends import_node_events.default {
     this.#tts = tts;
     this.#deferredValidation = new DeferredReplyValidation(
       this.#validateReplyIfPossible.bind(this),
-      this.#opts.minEndpointingDelay
+      this.#opts.minEndpointingDelay,
+      this,
+      this.#opts.turnDetector
     );
   }
   get fncCtx() {
@@ -327,13 +333,48 @@ class VoicePipelineAgent extends import_node_events.default {
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
     this.#humanInput.on(import_human_input.HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = (0, import_node_crypto.randomUUID)();
+      }
       this.#transcribedInterimText = event.alternatives[0].text;
+      this.#room.localParticipant.publishTranscription({
+        participantIdentity: this.#humanInput.participant.identity,
+        trackSid: this.#humanInput.subscribedTrack.sid,
+        segments: [
+          {
+            text: this.#transcribedInterimText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: ""
+          }
+        ]
+      });
     });
     this.#humanInput.on(import_human_input.HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
       const newTranscript = event.alternatives[0].text;
       if (!newTranscript) return;
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = (0, import_node_crypto.randomUUID)();
+      }
       this.#lastFinalTranscriptTime = Date.now();
-      this.#transcribedText += (this.#transcribedText ? " " : "") + newTranscript;
+      this.transcribedText += (this.transcribedText ? " " : "") + newTranscript;
+      this.#room.localParticipant.publishTranscription({
+        participantIdentity: this.#humanInput.participant.identity,
+        trackSid: this.#humanInput.subscribedTrack.sid,
+        segments: [
+          {
+            text: this.transcribedText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: ""
+          }
+        ]
+      });
+      this.#transcriptionId = void 0;
       if (this.#opts.preemptiveSynthesis && (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
         this.#synthesizeAgentReply();
       }
@@ -384,7 +425,7 @@ class VoicePipelineAgent extends import_node_events.default {
     this.#pendingAgentReply = import_speech_handle.SpeechHandle.createAssistantReply(
       this.#opts.allowInterruptions,
       true,
-      this.#transcribedText
+      this.transcribedText
     );
     const newHandle = this.#pendingAgentReply;
     this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
@@ -461,7 +502,7 @@ class VoicePipelineAgent extends import_node_events.default {
       const userMsg = import_llm2.ChatMessage.create({ text: userQuestion, role: import_llm2.ChatRole.USER });
       this.chatCtx.messages.push(userMsg);
       this.emit(4 /* USER_SPEECH_COMMITTED */, userMsg);
-      this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+      this.transcribedText = this.transcribedText.slice(userQuestion.length);
       handle.markUserCommitted();
     };
     commitUserQuestionIfNeeded();
@@ -475,7 +516,7 @@ class VoicePipelineAgent extends import_node_events.default {
       if (handle.interrupted) break;
     }
     commitUserQuestionIfNeeded();
-    const collectedText = handle.synthesisHandle.text;
+    let collectedText = this.#agentTranscribedText;
     const isUsingTools = handle.source instanceof import_llm.LLMStream && !!handle.source.functionCalls.length;
     const interrupted = handle.interrupted;
     if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
@@ -483,7 +524,7 @@ class VoicePipelineAgent extends import_node_events.default {
         this.chatCtx.messages.push(...handle.extraToolsMessages);
       }
       if (interrupted) {
-        collectedText + "\u2026";
+        collectedText += "\u2026";
       }
       const msg = import_llm2.ChatMessage.create({ text: collectedText, role: import_llm2.ChatRole.ASSISTANT });
       this.chatCtx.messages.push(msg);
@@ -578,6 +619,15 @@ class VoicePipelineAgent extends import_node_events.default {
     handle.setDone();
   }
   #synthesizeAgentSpeech(speechId, source) {
+    const synchronizer = new import_transcription.TextAudioSynchronizer(import_transcription.defaultTextSyncOptions);
+    synchronizer.on("textUpdated", (text) => {
+      this.#agentTranscribedText = text.text;
+      this.#room.localParticipant.publishTranscription({
+        participantIdentity: this.#room.localParticipant.identity,
+        trackSid: this.#agentPublication.sid,
+        segments: [text]
+      });
+    });
     if (!this.#agentOutput) {
       throw new Error("agent output should be initialized when ready");
     }
@@ -591,7 +641,7 @@ class VoicePipelineAgent extends import_node_events.default {
     if (!ttsSource) {
       throw new Error("beforeTTSCallback must return string or AsyncIterable<string>");
     }
-    return this.#agentOutput.synthesize(speechId, ttsSource);
+    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);
   }
   async #validateReplyIfPossible() {
     if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {
@@ -599,7 +649,7 @@ class VoicePipelineAgent extends import_node_events.default {
       return;
     }
     if (!this.#pendingAgentReply) {
-      if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
+      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {
         return;
       }
       this.#synthesizeAgentReply();
@@ -681,6 +731,7 @@ class DeferredReplyValidation {
   PUNCTUATION_REDUCE_FACTOR = 0.75;
   LATE_TRANSCRIPT_TOLERANCE = 1.5;
   // late compared to end of speech
+  UNLIKELY_ENDPOINT_DELAY = 6e3;
   #validateFunc;
   #validatingPromise;
   #validatingFuture = new import_utils.Future();
@@ -689,11 +740,15 @@ class DeferredReplyValidation {
   #speaking = false;
   #endOfSpeechDelay;
   #finalTranscriptDelay;
+  #turnDetector;
+  #agent;
   #abort;
-  constructor(validateFunc, minEndpointingDelay) {
+  constructor(validateFunc, minEndpointingDelay, agent, turnDetector) {
     this.#validateFunc = validateFunc;
     this.#endOfSpeechDelay = minEndpointingDelay;
     this.#finalTranscriptDelay = minEndpointingDelay;
+    this.#agent = agent;
+    this.#turnDetector = turnDetector;
   }
   get validating() {
     return !this.#validatingFuture.done;
@@ -733,7 +788,17 @@ class DeferredReplyValidation {
   }
   #run(delay) {
     var _a;
-    const runTask = async (delay2, signal) => {
+    const runTask = async (delay2, chatCtx, signal) => {
+      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {
+        const startTime = Date.now();
+        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);
+        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;
+        const elapsed = Date.now() - startTime;
+        if (eotProb < unlikelyThreshold) {
+          delay2 = this.UNLIKELY_ENDPOINT_DELAY;
+        }
+        delay2 = Math.max(0, delay2 - elapsed);
+      }
       const timeout = setTimeout(() => {
         this.#resetStates();
         this.#validateFunc();
@@ -745,7 +810,9 @@ class DeferredReplyValidation {
     (_a = this.#abort) == null ? void 0 : _a.abort();
     this.#abort = new AbortController();
     this.#validatingFuture = new import_utils.Future();
-    this.#validatingPromise = runTask(delay, this.#abort.signal);
+    const detectCtx = this.#agent.chatCtx.copy();
+    detectCtx.append({ text: this.#agent.transcribedText, role: import_llm2.ChatRole.USER });
+    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);
   }
 }
 // Annotate the CommonJS export names for ESM import in node: