@livekit/agents 0.4.6 → 0.5.0

package/dist/multimodal/agent_playout.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent_playout.js","sourceRoot":"","sources":["../../src/multimodal/agent_playout.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAA2B,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpG,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;AAExB,MAAM,OAAO,aAAc,SAAQ,YAAY;IAC7C,YAAY,CAAc;IAC1B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,aAAa,CAAS;IACtB,gBAAgB;IAChB,gBAAgB,CAAyB;IACzC,gBAAgB;IAChB,OAAO,CAAS;IAChB,gBAAgB;IAChB,MAAM,CAAS;IACf,gBAAgB;IAChB,YAAY,CAAU;IACtB,gBAAgB;IAChB,cAAc,CAAS;IACvB,gBAAgB;IAChB,eAAe,CAAqB,CAAC,2BAA2B;IAEhE,YACE,WAAwB,EACxB,UAAkB,EAClB,MAAc,EACd,YAAoB,EACpB,gBAAwC;QAExC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7D,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CACf,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CACrF,CAAC;IACJ,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC;IACrD,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC;IAChD,CAAC;IAED,SAAS;QACP,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAC9B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,CAAc;IAC1B,YAAY,CAAkC;IAC9C,WAAW,CAAS;IACpB,YAAY,CAAS;IACrB,YAAY,CAAS;IACrB,aAAa,CAAS;IACtB,YACE,WAAwB,EACxB,UAAkB,EAClB,WAAmB,EACnB,WAAmB,EACnB,YAAoB;QAEpB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACpC,CAAC;IAED,IAAI,CACF,MAAc,EACd,YAAoB,EACpB,gBAAwC,EACxC,UAAsC,EACtC,WAA2C;QAE3C,MAAM,MAAM,GAAG,IAAI,aAAa,CAC9B,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,EAChB,MAAM,EACN,YAAY,EACZ,gBAAgB,CACjB,CAAC;QACF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QAC9F,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,gBAAgB,CACd,OAAwC,EACxC,MAAqB,EACrB,UAAsC,EACtC,WAA2C;QAE3C,OAAO,IAAI,kBAAkB,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE;YAChE,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,QAAQ,CAAC,GAAG,EAAE;gBACZ,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,CAAC,KAAK,IAAI,EAAE;gBACV,IAAI,CAAC;oBACH,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAClC,CAAC;oBAED,IAAI,UAAU,GAAG,IAAI,CAAC;oBAEtB,MAAM,QAAQ,GAAG,GAAG,EAAE,CACpB,IAAI,kBAAkB,CAAO,CAAC,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE;wBACrE,IAAI,aAAa,GAAG,KAAK,CAAC;wBAC1B,YAAY,CAAC,GAAG,EAAE;4BAChB,aAAa,GAAG,IAAI,CAAC;wBACvB,CAAC,CAAC,CAAC;wBAEH,CAAC,KAAK,IAAI,EAAE;4BACV,IAAI,CAAC;gCACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oCACpC,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;wCAC/B,MAAM;oCACR,CAAC;oCACD,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gCACzC,CAAC;gCACD,WAAW,EAAE,CAAC;4BAChB,CAAC;4BAAC,OAAO,KAAK,EAAE,CAAC;gCACf,UAAU,CAAC,KAAK,CAAC,CAAC;4BACpB,CAAC;wBACH,CAAC,CAAC,EAAE,CAAC;oBACP,CAAC,CAAC,CAAC;oBAEL,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB,IAAI,kBAAkB,CAAO,CAAC,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,EAAE;wBAC9E,IAAI,gBAAgB,GAAG,KAAK,CAAC;wBAC7B,eAAe,CAAC,GAAG,EAAE;4BACnB,gBAAgB,GAAG,IAAI,CAAC;wBAC1B,CAAC,CAAC,CAAC;wBAEH,CAAC,KAAK,IAAI,EAAE;4BACV,IAAI,CAAC;gCACH,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,CAAC;gCAC7C,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,YAAY,EACjB,iBAAiB,CAClB,CAAC;gCAEF,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oCACtC,IAAI,gBAAgB,IAAI,SAAS,EAAE,CAAC;wCAClC,MAAM;oCACR,CAAC;oCACD,IAAI,UAAU,EAAE,CAAC;wCACf,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;wCAChC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;wCAC7B,UAAU,GAAG,KAAK,CAAC;oCACrB,CAAC;oCAED,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oCAEzC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;wCACjD,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;wCACrE,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oCAC1C,CAAC;gCACH,CAAC;gCAED,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC;oCACpC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;wCAChC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;wCACrE,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oCAC1C,CAAC;oCAED,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;oCAE5C,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;gCAC3C,CAAC;gCAED,cAAc,EAAE,CAAC;4BACnB,CAAC;4BAAC,OAAO,KAAK,EAAE,CAAC;gCACf,aAAa,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;wBACH,CAAC,CAAC,EAAE,CAAC;oBACP,CAAC,CAAC,CAAC;oBAEL,MAAM,YAAY,GAAG,QAAQ,EAAE,CAAC;oBAChC,MAAM,WAAW,GAAG,OAAO,EAAE,CAAC;oBAE9B,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzD,CAAC;4BAAS,CAAC;wBACT,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;4BAC7B,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;wBACtC,CAAC;wBAED,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC;wBAElF,IAAI,MAAM,CAAC,WAAW,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;4BAC5C,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,wCAAwC;wBAC1E,CAAC;wBAED,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;4BAC9B,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAC;wBACvC,CAAC;wBAED,IAAI,CAAC,UAAU,EAAE,CAAC;4BAChB,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gCACxB,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC;4BAC7C,CAAC;4BAED,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;wBACnD,CAAC;wBAED,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;wBACzB,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;oBAC1D,CAAC;oBAED,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"sources":["../../src/multimodal/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { type AudioSource } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport type { TranscriptionForwarder } from '../transcription.js';\nimport { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\n\nexport const proto = {};\n\nexport class PlayoutHandle extends EventEmitter {\n  #audioSource: AudioSource;\n  #sampleRate: number;\n  #itemId: string;\n  #contentIndex: number;\n  /** @internal */\n  transcriptionFwd: TranscriptionForwarder;\n  /** @internal */\n  doneFut: Future;\n  /** @internal */\n  intFut: Future;\n  /** @internal */\n  #interrupted: boolean;\n  /** @internal */\n  pushedDuration: number;\n  /** @internal */\n  totalPlayedTime: number | undefined; // Set when playout is done\n\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    itemId: string,\n    contentIndex: number,\n    transcriptionFwd: TranscriptionForwarder,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#sampleRate = sampleRate;\n    this.#itemId = itemId;\n    this.#contentIndex = contentIndex;\n    this.transcriptionFwd = transcriptionFwd;\n    this.doneFut = new Future();\n    this.intFut = new Future();\n    this.#interrupted = false;\n    this.pushedDuration = 0;\n    this.totalPlayedTime = undefined;\n  }\n\n  get itemId(): string {\n    return this.#itemId;\n  }\n\n  get audioSamples(): number {\n    if (this.totalPlayedTime !== undefined) {\n      return Math.floor(this.totalPlayedTime * this.#sampleRate);\n    }\n\n    return Math.floor(\n      (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),\n    );\n  }\n\n  get textChars(): number {\n    return this.transcriptionFwd.currentCharacterIndex;\n  }\n\n  get contentIndex(): number {\n    return this.#contentIndex;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.doneFut.done) return;\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n}\n\nexport class AgentPlayout extends EventEmitter {\n  #audioSource: AudioSource;\n  #playoutTask: CancellablePromise<void> | null;\n  #sampleRate: number;\n  #numChannels: number;\n  #inFrameSize: number;\n  #outFrameSize: number;\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    numChannels: number,\n    inFrameSize: number,\n    outFrameSize: number,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#playoutTask = null;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n    this.#inFrameSize = inFrameSize;\n    this.#outFrameSize = outFrameSize;\n  }\n\n  play(\n    itemId: string,\n    contentIndex: number,\n    transcriptionFwd: TranscriptionForwarder,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): PlayoutHandle {\n    const handle = new PlayoutHandle(\n      this.#audioSource,\n      this.#sampleRate,\n      itemId,\n      contentIndex,\n      transcriptionFwd,\n    );\n    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);\n    return handle;\n  }\n\n  #makePlayoutTask(\n    oldTask: CancellablePromise<void> | null,\n    handle: PlayoutHandle,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): CancellablePromise<void> {\n    return new CancellablePromise<void>((resolve, reject, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      (async () => {\n        try {\n          if (oldTask) {\n            await gracefullyCancel(oldTask);\n          }\n\n          let firstFrame = true;\n\n          const readText = () =>\n            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {\n              let cancelledText = false;\n              onCancelText(() => {\n                cancelledText = true;\n              });\n\n              (async () => {\n                try {\n                  for await (const text of textStream) {\n                    if (cancelledText || cancelled) {\n                      break;\n                    }\n                    handle.transcriptionFwd.pushText(text);\n                  }\n                  resolveText();\n                } catch (error) {\n                  rejectText(error);\n                }\n              })();\n            });\n\n          const capture = () =>\n            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {\n              let cancelledCapture = false;\n              onCancelCapture(() => {\n                cancelledCapture = true;\n              });\n\n              (async () => {\n                try {\n                  const samplesPerChannel = this.#outFrameSize;\n                  const bstream = new AudioByteStream(\n                    this.#sampleRate,\n                    this.#numChannels,\n                    samplesPerChannel,\n                  );\n\n                  for await (const frame of audioStream) {\n                    if (cancelledCapture || cancelled) {\n                      break;\n                    }\n                    if (firstFrame) {\n                      handle.transcriptionFwd.start();\n                      this.emit('playout_started');\n                      firstFrame = false;\n                    }\n\n                    handle.transcriptionFwd.pushAudio(frame);\n\n                    for (const f of bstream.write(frame.data.buffer)) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n                  }\n\n                  if (!cancelledCapture && !cancelled) {\n                    for (const f of bstream.flush()) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n\n                    handle.transcriptionFwd.markAudioComplete();\n\n                    await this.#audioSource.waitForPlayout();\n                  }\n\n                  resolveCapture();\n                } catch (error) {\n                  rejectCapture(error);\n                }\n              })();\n            });\n\n          const readTextTask = readText();\n          const captureTask = capture();\n\n          try {\n            await Promise.race([captureTask, handle.intFut.await]);\n          } finally {\n            if (!captureTask.isCancelled) {\n              await gracefullyCancel(captureTask);\n            }\n\n            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n            if (handle.interrupted || captureTask.error) {\n              this.#audioSource.clearQueue(); // make sure to remove any queued frames\n            }\n\n            if (!readTextTask.isCancelled) {\n              await gracefullyCancel(readTextTask);\n            }\n\n            if (!firstFrame) {\n              if (!handle.interrupted) {\n                handle.transcriptionFwd.markTextComplete();\n              }\n\n              this.emit('playout_stopped', handle.interrupted);\n            }\n\n            handle.doneFut.resolve();\n            await handle.transcriptionFwd.close(handle.interrupted);\n          }\n\n          resolve();\n        } catch (error) {\n          reject(error);\n        }\n      })();\n    });\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,uBAAuB;AAEhC,SAAkC,oBAAoB,QAAQ,wBAAwB;AAE/E,MAAM,QAAQ,CAAC;AAEf,MAAM,sBAAsB,aAAa;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA,YACE,aACA,YACA,QACA,cACA,kBACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,gBAAgB;AACrB,SAAK,mBAAmB;AACxB,SAAK,UAAU,IAAI,OAAO;AAC1B,SAAK,SAAS,IAAI,OAAO;AACzB,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,KAAK,MAAM,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3D;AAEA,WAAO,KAAK;AAAA,OACT,KAAK,iBAAiB,KAAK,aAAa,mBAAmB,KAAK,cAAc;AAAA,IACjF;AAAA,EACF;AAAA,EAEA,IAAI,YAAoB;AACtB,WAAO,KAAK,iBAAiB;AAAA,EAC/B;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,QAAQ,KAAM;AACvB,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AACF;AAEO,MAAM,qBAAqB,aAAa;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YACE,aACA,YACA,aACA,aACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,QACA,cACA,kBACA,YACA,aACe;AACf,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,SAAK,eAAe,KAAK,iBAAiB,KAAK,cAAc,QAAQ,YAAY,WAAW;AAC5F,WAAO;AAAA,EACT;AAAA,EAEA,iBACE,SACA,QACA,YACA,aAC0B;AAC1B,WAAO,IAAI,mBAAyB,CAAC,SAAS,QAAQ,aAAa;AACjE,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,OAAC,YAAY;AACX,YAAI;AACF,cAAI,SAAS;AACX,kBAAM,iBAAiB,OAAO;AAAA,UAChC;AAEA,cAAI,aAAa;AAEjB,gBAAM,WAAW,MACf,IAAI,mBAAyB,CAAC,aAAa,YAAY,iBAAiB;AACtE,gBAAI,gBAAgB;AACpB,yBAAa,MAAM;AACjB,8BAAgB;AAAA,YAClB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,iCAAiB,QAAQ,YAAY;AACnC,sBAAI,iBAAiB,WAAW;AAC9B;AAAA,kBACF;AACA,yBAAO,iBAAiB,SAAS,IAAI;AAAA,gBACvC;AACA,4BAAY;AAAA,cACd,SAAS,OAAO;AACd,2BAAW,KAAK;AAAA,cAClB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,UAAU,MACd,IAAI,mBAAyB,CAAC,gBAAgB,eAAe,oBAAoB;AAC/E,gBAAI,mBAAmB;AACvB,4BAAgB,MAAM;AACpB,iCAAmB;AAAA,YACrB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,sBAAM,oBAAoB,KAAK;AAC/B,sBAAM,UAAU,IAAI;AAAA,kBAClB,KAAK;AAAA,kBACL,KAAK;AAAA,kBACL;AAAA,gBACF;AAEA,iCAAiB,SAAS,aAAa;AACrC,sBAAI,oBAAoB,WAAW;AACjC;AAAA,kBACF;AACA,sBAAI,YAAY;AACd,2BAAO,iBAAiB,MAAM;AAC9B,yBAAK,KAAK,iBAAiB;AAC3B,iCAAa;AAAA,kBACf;AAEA,yBAAO,iBAAiB,UAAU,KAAK;AAEvC,6BAAW,KAAK,QAAQ,MAAM,MAAM,KAAK,MAAM,GAAG;AAChD,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAAA,gBACF;AAEA,oBAAI,CAAC,oBAAoB,CAAC,WAAW;AACnC,6BAAW,KAAK,QAAQ,MAAM,GAAG;AAC/B,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAEA,yBAAO,iBAAiB,kBAAkB;AAE1C,wBAAM,KAAK,aAAa,eAAe;AAAA,gBACzC;AAEA,+BAAe;AAAA,cACjB,SAAS,OAAO;AACd,8BAAc,KAAK;AAAA,cACrB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,eAAe,SAAS;AAC9B,gBAAM,cAAc,QAAQ;AAE5B,cAAI;AACF,kBAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,UACvD,UAAE;AACA,gBAAI,CAAC,YAAY,aAAa;AAC5B,oBAAM,iBAAiB,WAAW;AAAA,YACpC;AAEA,mBAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,gBAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,mBAAK,aAAa,WAAW;AAAA,YAC/B;AAEA,gBAAI,CAAC,aAAa,aAAa;AAC7B,oBAAM,iBAAiB,YAAY;AAAA,YACrC;AAEA,gBAAI,CAAC,YAAY;AACf,kBAAI,CAAC,OAAO,aAAa;AACvB,uBAAO,iBAAiB,iBAAiB;AAAA,cAC3C;AAEA,mBAAK,KAAK,mBAAmB,OAAO,WAAW;AAAA,YACjD;AAEA,mBAAO,QAAQ,QAAQ;AACvB,kBAAM,OAAO,iBAAiB,MAAM,OAAO,WAAW;AAAA,UACxD;AAEA,kBAAQ;AAAA,QACV,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,QACd;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/multimodal/index.cjs

@@ -0,0 +1,25 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var multimodal_exports = {};
+module.exports = __toCommonJS(multimodal_exports);
+__reExport(multimodal_exports, require("./multimodal_agent.cjs"), module.exports);
+__reExport(multimodal_exports, require("./agent_playout.cjs"), module.exports);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ...require("./multimodal_agent.cjs"),
+  ...require("./agent_playout.cjs")
+});
+//# sourceMappingURL=index.cjs.map

package/dist/multimodal/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/multimodal/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport * from './multimodal_agent.js';\nexport * from './agent_playout.js';\n"],"mappings":";;;;;;;;;;;;;;;AAAA;AAAA;AAGA,+BAAc,kCAHd;AAIA,+BAAc,+BAJd;","names":[]}

package/dist/multimodal/index.js

@@ -1,6 +1,3 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-export * from './multimodal_agent.js';
-export * from './agent_playout.js';
+export * from "./multimodal_agent.js";
+export * from "./agent_playout.js";
 //# sourceMappingURL=index.js.map

package/dist/multimodal/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/multimodal/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC"}
+{"version":3,"sources":["../../src/multimodal/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport * from './multimodal_agent.js';\nexport * from './agent_playout.js';\n"],"mappings":"AAGA,cAAc;AACd,cAAc;","names":[]}

package/dist/multimodal/multimodal_agent.cjs

@@ -0,0 +1,404 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var multimodal_agent_exports = {};
+__export(multimodal_agent_exports, {
+  AGENT_STATE_ATTRIBUTE: () => AGENT_STATE_ATTRIBUTE,
+  MultimodalAgent: () => MultimodalAgent,
+  RealtimeModel: () => RealtimeModel,
+  RealtimeSession: () => RealtimeSession
+});
+module.exports = __toCommonJS(multimodal_agent_exports);
+var import_rtc_node = require("@livekit/rtc-node");
+var import_node_events = require("node:events");
+var import_audio = require("../audio.cjs");
+var llm = __toESM(require("../llm/index.cjs"), 1);
+var import_log = require("../log.cjs");
+var import_transcription = require("../transcription.cjs");
+var import_utils = require("../utils.cjs");
+var import_agent_playout = require("./agent_playout.cjs");
+class RealtimeSession extends import_node_events.EventEmitter {
+}
+class RealtimeModel {
+}
+const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
+class MultimodalAgent extends import_node_events.EventEmitter {
+  model;
+  room = null;
+  linkedParticipant = null;
+  subscribedTrack = null;
+  readMicroTask = null;
+  constructor({
+    model,
+    chatCtx,
+    fncCtx
+  }) {
+    super();
+    this.model = model;
+    this.#chatCtx = chatCtx;
+    this.#fncCtx = fncCtx;
+  }
+  #participant = null;
+  #agentPublication = null;
+  #localTrackSid = null;
+  #localSource = null;
+  #agentPlayout = null;
+  #playingHandle = void 0;
+  #logger = (0, import_log.log)();
+  #session = null;
+  #fncCtx = void 0;
+  #chatCtx = void 0;
+  #_started = false;
+  #_pendingFunctionCalls = /* @__PURE__ */ new Set();
+  #_speaking = false;
+  get fncCtx() {
+    return this.#fncCtx;
+  }
+  set fncCtx(ctx) {
+    this.#fncCtx = ctx;
+    if (this.#session) {
+      this.#session.fncCtx = ctx;
+    }
+  }
+  get #pendingFunctionCalls() {
+    return this.#_pendingFunctionCalls;
+  }
+  set #pendingFunctionCalls(calls) {
+    this.#_pendingFunctionCalls = calls;
+    this.#updateState();
+  }
+  get #speaking() {
+    return this.#_speaking;
+  }
+  set #speaking(isSpeaking) {
+    this.#_speaking = isSpeaking;
+    this.#updateState();
+  }
+  get #started() {
+    return this.#_started;
+  }
+  set #started(started) {
+    this.#_started = started;
+    this.#updateState();
+  }
+  start(room, participant = null) {
+    return new Promise(async (resolve, reject) => {
+      var _a;
+      if (this.#started) {
+        reject(new Error("MultimodalAgent already started"));
+      }
+      this.#updateState();
+      room.on(import_rtc_node.RoomEvent.ParticipantConnected, (participant2) => {
+        if (this.linkedParticipant) {
+          return;
+        }
+        this.#linkParticipant(participant2.identity);
+      });
+      room.on(
+        import_rtc_node.RoomEvent.TrackPublished,
+        (trackPublication, participant2) => {
+          if (this.linkedParticipant && participant2.identity === this.linkedParticipant.identity && trackPublication.source === import_rtc_node.TrackSource.SOURCE_MICROPHONE && !trackPublication.subscribed) {
+            trackPublication.setSubscribed(true);
+          }
+        }
+      );
+      room.on(import_rtc_node.RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));
+      this.room = room;
+      this.#participant = participant;
+      this.#localSource = new import_rtc_node.AudioSource(this.model.sampleRate, this.model.numChannels);
+      this.#agentPlayout = new import_agent_playout.AgentPlayout(
+        this.#localSource,
+        this.model.sampleRate,
+        this.model.numChannels,
+        this.model.inFrameSize,
+        this.model.outFrameSize
+      );
+      const onPlayoutStarted = () => {
+        this.emit("agent_started_speaking");
+        this.#speaking = true;
+      };
+      const onPlayoutStopped = (interrupted) => {
+        this.emit("agent_stopped_speaking");
+        this.#speaking = false;
+        if (this.#playingHandle) {
+          let text = this.#playingHandle.transcriptionFwd.text;
+          if (interrupted) {
+            text += "\u2026";
+          }
+          const msg = llm.ChatMessage.create({
+            role: llm.ChatRole.ASSISTANT,
+            text
+          });
+          if (interrupted) {
+            this.emit("agent_speech_interrupted", msg);
+          } else {
+            this.emit("agent_speech_committed", msg);
+          }
+          this.#logger.child({ transcription: text, interrupted }).debug("committed agent speech");
+        }
+      };
+      this.#agentPlayout.on("playout_started", onPlayoutStarted);
+      this.#agentPlayout.on("playout_stopped", onPlayoutStopped);
+      const track = import_rtc_node.LocalAudioTrack.createAudioTrack("assistant_voice", this.#localSource);
+      const options = new import_rtc_node.TrackPublishOptions();
+      options.source = import_rtc_node.TrackSource.SOURCE_MICROPHONE;
+      this.#agentPublication = await ((_a = room.localParticipant) == null ? void 0 : _a.publishTrack(track, options)) || null;
+      if (!this.#agentPublication) {
+        this.#logger.error("Failed to publish track");
+        reject(new Error("Failed to publish track"));
+        return;
+      }
+      await this.#agentPublication.waitForSubscription();
+      if (participant) {
+        if (typeof participant === "string") {
+          this.#linkParticipant(participant);
+        } else {
+          this.#linkParticipant(participant.identity);
+        }
+      } else {
+        for (const participant2 of room.remoteParticipants.values()) {
+          this.#linkParticipant(participant2.identity);
+          break;
+        }
+      }
+      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });
+      this.#started = true;
+      this.#session.on("response_content_added", (message) => {
+        var _a2;
+        const trFwd = new import_transcription.BasicTranscriptionForwarder(
+          this.room,
+          this.room.localParticipant.identity,
+          this.#getLocalTrackSid(),
+          message.responseId
+        );
+        const handle = (_a2 = this.#agentPlayout) == null ? void 0 : _a2.play(
+          message.itemId,
+          message.contentIndex,
+          trFwd,
+          message.textStream,
+          message.audioStream
+        );
+        this.#playingHandle = handle;
+      });
+      this.#session.on("input_speech_committed", (ev) => {
+        var _a2, _b;
+        const participantIdentity = (_a2 = this.linkedParticipant) == null ? void 0 : _a2.identity;
+        const trackSid = (_b = this.subscribedTrack) == null ? void 0 : _b.sid;
+        if (participantIdentity && trackSid) {
+          this.#publishTranscription(participantIdentity, trackSid, "\u2026", false, ev.itemId);
+        } else {
+          this.#logger.error("Participant or track not set");
+        }
+      });
+      this.#session.on("input_speech_transcription_completed", (ev) => {
+        var _a2, _b;
+        const transcription = ev.transcript;
+        const participantIdentity = (_a2 = this.linkedParticipant) == null ? void 0 : _a2.identity;
+        const trackSid = (_b = this.subscribedTrack) == null ? void 0 : _b.sid;
+        if (participantIdentity && trackSid) {
+          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);
+        } else {
+          this.#logger.error("Participant or track not set");
+        }
+        const userMsg = llm.ChatMessage.create({
+          role: llm.ChatRole.USER,
+          text: transcription
+        });
+        this.emit("user_speech_committed", userMsg);
+        this.#logger.child({ transcription }).debug("committed user speech");
+      });
+      this.#session.on("input_speech_started", (ev) => {
+        var _a2, _b;
+        if (this.#playingHandle && !this.#playingHandle.done) {
+          this.#playingHandle.interrupt();
+          this.#session.conversation.item.truncate(
+            this.#playingHandle.itemId,
+            this.#playingHandle.contentIndex,
+            Math.floor(this.#playingHandle.audioSamples / 24e3 * 1e3)
+          );
+          this.#playingHandle = void 0;
+        }
+        const participantIdentity = (_a2 = this.linkedParticipant) == null ? void 0 : _a2.identity;
+        const trackSid = (_b = this.subscribedTrack) == null ? void 0 : _b.sid;
+        if (participantIdentity && trackSid) {
+          this.#publishTranscription(participantIdentity, trackSid, "\u2026", false, ev.itemId);
+        }
+      });
+      this.#session.on("input_speech_stopped", (ev) => {
+        this.emit("user_stopped_speaking");
+      });
+      this.#session.on("function_call_started", (ev) => {
+        this.#pendingFunctionCalls.add(ev.callId);
+        this.#updateState();
+      });
+      this.#session.on("function_call_completed", (ev) => {
+        this.#pendingFunctionCalls.delete(ev.callId);
+        this.#updateState();
+      });
+      this.#session.on("function_call_failed", (ev) => {
+        this.#pendingFunctionCalls.delete(ev.callId);
+        this.#updateState();
+      });
+      resolve(this.#session);
+    });
+  }
+  #linkParticipant(participantIdentity) {
+    if (!this.room) {
+      this.#logger.error("Room is not set");
+      return;
+    }
+    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;
+    if (!this.linkedParticipant) {
+      this.#logger.error(`Participant with identity ${participantIdentity} not found`);
+      return;
+    }
+    if (this.linkedParticipant.trackPublications.size > 0) {
+      this.#subscribeToMicrophone();
+    }
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+      if (publication.source === import_rtc_node.TrackSource.SOURCE_MICROPHONE && publication.track) {
+        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);
+        break;
+      }
+    }
+  }
+  #subscribeToMicrophone() {
+    if (!this.linkedParticipant) {
+      this.#logger.error("Participant is not set");
+      return;
+    }
+    let microphonePublication = void 0;
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+      if (publication.source === import_rtc_node.TrackSource.SOURCE_MICROPHONE) {
+        microphonePublication = publication;
+        break;
+      }
+    }
+    if (!microphonePublication) {
+      return;
+    }
+    if (!microphonePublication.subscribed) {
+      microphonePublication.setSubscribed(true);
+    }
+  }
+  #handleTrackSubscription(track, publication, participant) {
+    var _a;
+    if (publication.source !== import_rtc_node.TrackSource.SOURCE_MICROPHONE || participant.identity !== ((_a = this.linkedParticipant) == null ? void 0 : _a.identity)) {
+      return;
+    }
+    const readAudioStreamTask = async (audioStream) => {
+      const bstream = new import_audio.AudioByteStream(
+        this.model.sampleRate,
+        this.model.numChannels,
+        this.model.inFrameSize
+      );
+      for await (const frame of audioStream) {
+        const audioData = frame.data;
+        for (const frame2 of bstream.write(audioData.buffer)) {
+          this.#session.inputAudioBuffer.append(frame2);
+        }
+      }
+    };
+    this.subscribedTrack = track;
+    if (this.readMicroTask) {
+      this.readMicroTask.cancel();
+    }
+    let cancel;
+    this.readMicroTask = {
+      promise: new Promise((resolve, reject) => {
+        cancel = () => {
+          reject(new Error("Task cancelled"));
+        };
+        readAudioStreamTask(new import_rtc_node.AudioStream(track, this.model.sampleRate, this.model.numChannels)).then(resolve).catch(reject);
+      }),
+      cancel: () => cancel()
+    };
+  }
+  #getLocalTrackSid() {
+    var _a;
+    if (!this.#localTrackSid && this.room && this.room.localParticipant) {
+      this.#localTrackSid = (0, import_utils.findMicroTrackId)(this.room, (_a = this.room.localParticipant) == null ? void 0 : _a.identity);
+    }
+    return this.#localTrackSid;
+  }
+  #publishTranscription(participantIdentity, trackSid, text, isFinal, id) {
+    var _a;
+    this.#logger.debug(
+      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`
+    );
+    if (!((_a = this.room) == null ? void 0 : _a.localParticipant)) {
+      this.#logger.error("Room or local participant not set");
+      return;
+    }
+    this.room.localParticipant.publishTranscription({
+      participantIdentity,
+      trackSid,
+      segments: [
+        {
+          text,
+          final: isFinal,
+          id,
+          startTime: BigInt(0),
+          endTime: BigInt(0),
+          language: ""
+        }
+      ]
+    });
+  }
+  #updateState() {
+    let newState = "initializing";
+    if (this.#pendingFunctionCalls.size > 0) {
+      newState = "thinking";
+    } else if (this.#speaking) {
+      newState = "speaking";
+    } else if (this.#started) {
+      newState = "listening";
+    }
+    this.#setState(newState);
+  }
+  #setState(state) {
+    var _a;
+    if (((_a = this.room) == null ? void 0 : _a.isConnected) && this.room.localParticipant) {
+      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+      if (currentState !== state) {
+        this.room.localParticipant.setAttributes({
+          [AGENT_STATE_ATTRIBUTE]: state
+        });
+        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);
+      }
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AGENT_STATE_ATTRIBUTE,
+  MultimodalAgent,
+  RealtimeModel,
+  RealtimeSession
+});
+//# sourceMappingURL=multimodal_agent.cjs.map

package/dist/multimodal/multimodal_agent.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: { promise: Promise<void>; cancel: () => void } | null = null;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    if (this.readMicroTask) {\n      this.readMicroTask.cancel();\n    }\n\n    let cancel: () => void;\n    this.readMicroTask = {\n      promise: new Promise<void>((resolve, reject) => {\n        cancel = () => {\n          reject(new Error('Task cancelled'));\n        };\n        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n          .then(resolve)\n          .catch(reject);\n      }),\n      cancel: () => cancel(),\n    };\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAOO;AACP,yBAA6B;AAC7B,mBAAgC;AAChC,UAAqB;AACrB,iBAAoB;AACpB,2BAA4C;AAC5C,mBAAiC;AACjC,2BAAiD;AAM1C,MAAe,wBAAwB,gCAAa;AAM3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,gCAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAuE;AAAA,EAEvE,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,cAAU,gBAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAxIlD;AAyIM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,0BAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAQ;AAAA,MAC5C,CAAC;AACD,WAAK;AAAA,QACH,0BAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,4BAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,0BAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,4BAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oCAAoB;AACxC,cAAQ,SAAS,4BAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAQ;AAAA,QAC5C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAQ;AAC1C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AA3OnE,YAAAC;AA6OQ,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AA/P9D,YAAAA,KAAA;AAiQQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AA3Q5E,YAAAA,KAAA;AA6QQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AA7R5D,YAAAA,KAAA;AA8RQ,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AA/XJ;AAgYI,QACE,YAAY,WAAW,4BAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,QAAI;AACJ,SAAK,gBAAgB;AAAA,MACnB,SAAS,IAAI,QAAc,CAAC,SAAS,WAAW;AAC9C,iBAAS,MAAM;AACb,iBAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,QACpC;AACA,4BAAoB,IAAI,4BAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,MACjB,CAAC;AAAA,MACD,QAAQ,MAAM,OAAO;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,oBAAmC;AAxarC;AAyaI,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,qBAAiB,+BAAiB,KAAK,OAAM,UAAK,KAAK,qBAAV,mBAA4B,QAAQ;AAAA,IACxF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AArbV;AAsbI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AA3d/B;AA4dI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAW,qBAAqB;AAChF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}

package/dist/multimodal/multimodal_agent.d.ts

@@ -1,4 +1,4 @@
-/// <reference types="node" />
+/// <reference types="node" resolution-mode="require"/>
 import type { RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
 import { EventEmitter } from 'node:events';
 import * as llm from '../llm/index.js';