@livekit/agents 0.6.4 → 0.7.1

package/dist/job.js

@@ -1,5 +1,14 @@
 import { ParticipantKind, RoomEvent, TrackKind } from "@livekit/rtc-node";
 import { log } from "./log.js";
+class CurrentJobContext {
+  static #current;
+  constructor(proc) {
+    CurrentJobContext.#current = proc;
+  }
+  static getCurrent() {
+    return CurrentJobContext.#current;
+  }
+}
 var AutoSubscribe = /* @__PURE__ */ ((AutoSubscribe2) => {
   AutoSubscribe2[AutoSubscribe2["SUBSCRIBE_ALL"] = 0] = "SUBSCRIBE_ALL";
   AutoSubscribe2[AutoSubscribe2["SUBSCRIBE_NONE"] = 1] = "SUBSCRIBE_NONE";
@@ -24,7 +33,8 @@ class JobContext {
   #participantEntrypoints = [];
   #participantTasks = {};
   #logger;
-  constructor(proc, info, room, onConnect, onShutdown) {
+  #inferenceExecutor;
+  constructor(proc, info, room, onConnect, onShutdown, inferenceExecutor) {
     this.#proc = proc;
     this.#info = info;
     this.#room = room;
@@ -33,6 +43,7 @@ class JobContext {
     this.onParticipantConnected = this.onParticipantConnected.bind(this);
     this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
     this.#logger = log().child({ info: this.#info });
+    this.#inferenceExecutor = inferenceExecutor;
   }
   get proc() {
     return this.#proc;
@@ -48,6 +59,10 @@ class JobContext {
   get agent() {
     return this.#room.localParticipant;
   }
+  /** @returns The global inference executor */
+  get inferenceExecutor() {
+    return this.#inferenceExecutor;
+  }
   /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
   addShutdownCallback(callback) {
     this.shutdownCallbacks.push(callback);
@@ -194,6 +209,7 @@ class JobRequest {
 }
 export {
   AutoSubscribe,
+  CurrentJobContext,
   FunctionExistsError,
   JobContext,
   JobProcess,

package/dist/job.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/job.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as proto from '@livekit/protocol';\nimport type {\n  E2EEOptions,\n  LocalParticipant,\n  RemoteParticipant,\n  Room,\n  RtcConfiguration,\n} from '@livekit/rtc-node';\nimport { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';\nimport type { Logger } from 'pino';\nimport { log } from './log.js';\n\n/** Which tracks, if any, should the agent automatically subscribe to? */\nexport enum AutoSubscribe {\n  SUBSCRIBE_ALL,\n  SUBSCRIBE_NONE,\n  VIDEO_ONLY,\n  AUDIO_ONLY,\n}\n\nexport type JobAcceptArguments = {\n  name: string;\n  identity: string;\n  metadata: string;\n  attributes?: { [key: string]: string };\n};\n\nexport type RunningJobInfo = {\n  acceptArguments: JobAcceptArguments;\n  job: proto.Job;\n  url: string;\n  token: string;\n};\n\n/** Attempted to add a function callback, but the function already exists. */\nexport class FunctionExistsError extends Error {\n  constructor(msg?: string) {\n    super(msg);\n    Object.setPrototypeOf(this, new.target.prototype);\n  }\n}\n\n/** The job and environment context as seen by the agent, accessible by the entrypoint function. */\nexport class JobContext {\n  #proc: JobProcess;\n  #info: RunningJobInfo;\n  #room: Room;\n  #onConnect: () => void;\n  #onShutdown: (s: string) => void;\n  /** @internal */\n  shutdownCallbacks: (() => Promise<void>)[] = [];\n  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];\n  #participantTasks: {\n    [id: string]: {\n      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;\n      result: Promise<void>;\n    };\n  } = {};\n  #logger: Logger;\n\n  constructor(\n    proc: JobProcess,\n    info: RunningJobInfo,\n    room: Room,\n    onConnect: () => void,\n    onShutdown: (s: string) => void,\n  ) {\n    this.#proc = proc;\n    this.#info = info;\n    this.#room = room;\n    this.#onConnect = onConnect;\n    this.#onShutdown = onShutdown;\n    this.onParticipantConnected = this.onParticipantConnected.bind(this);\n    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);\n    this.#logger = log().child({ info: this.#info });\n  }\n\n  get proc(): JobProcess {\n    return this.#proc;\n  }\n\n  get job(): proto.Job {\n    return this.#info.job;\n  }\n\n  /** @returns The room the agent was called into */\n  get room(): Room {\n    return this.#room;\n  }\n\n  /** @returns The agent's participant if connected to the room, otherwise `undefined` */\n  get agent(): LocalParticipant | undefined {\n    return this.#room.localParticipant;\n  }\n\n  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */\n  addShutdownCallback(callback: () => Promise<void>) {\n    this.shutdownCallbacks.push(callback);\n  }\n\n  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {\n    if (!this.#room.isConnected) {\n      throw new Error('room is not connected');\n    }\n\n    for (const p of this.#room.remoteParticipants.values()) {\n      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {\n        return p;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const onParticipantConnected = (participant: RemoteParticipant) => {\n        if (\n          (!identity || participant.identity === identity) &&\n          participant.info.kind != ParticipantKind.AGENT\n        ) {\n          clearHandlers();\n          resolve(participant);\n        }\n      };\n      const onDisconnected = () => {\n        clearHandlers();\n        reject(new Error('Room disconnected while waiting for participant'));\n      };\n\n      const clearHandlers = () => {\n        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);\n        this.#room.off(RoomEvent.Disconnected, onDisconnected);\n      };\n\n      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);\n      this.#room.on(RoomEvent.Disconnected, onDisconnected);\n    });\n  }\n\n  /**\n   * Connects the agent to the room.\n   *\n   * @remarks\n   * It is recommended to run this command as early in the function as possible, as executing it\n   * later may cause noticeable delay between user and agent joins.\n   *\n   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |\n   * @livekit/rtc-node} for more information about the parameters.\n   */\n  async connect(\n    e2ee?: E2EEOptions,\n    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,\n    rtcConfig?: RtcConfiguration,\n  ) {\n    const opts = {\n      e2ee,\n      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,\n      rtcConfig,\n      dynacast: false,\n    };\n\n    await this.#room.connect(this.#info.url, this.#info.token, opts);\n    this.#onConnect();\n\n    this.#room.remoteParticipants.forEach(this.onParticipantConnected);\n\n    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {\n      this.#room.remoteParticipants.forEach((p) => {\n        p.trackPublications.forEach((pub) => {\n          if (\n            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||\n            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)\n          ) {\n            pub.setSubscribed(true);\n          }\n        });\n      });\n    }\n  }\n\n  /**\n   * Gracefully shuts down the job, and runs all shutdown promises.\n   *\n   * @param reason - Optional reason for shutdown\n   */\n  shutdown(reason = '') {\n    this.#onShutdown(reason);\n  }\n\n  /** @internal */\n  onParticipantConnected(p: RemoteParticipant) {\n    for (const callback of this.#participantEntrypoints) {\n      if (this.#participantTasks[p.identity!]?.callback == callback) {\n        this.#logger.warn(\n          'a participant has joined before a prior prticipant task matching the same identity has finished:',\n          p.identity,\n        );\n      }\n      const result = callback(this, p);\n      result.finally(() => delete this.#participantTasks[p.identity!]);\n      this.#participantTasks[p.identity!] = { callback, result };\n    }\n  }\n\n  /**\n   * Adds a promise to be awaited whenever a new participant joins the room.\n   *\n   * @throws {@link FunctionExistsError} if an entrypoint already exists\n   */\n  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {\n    if (this.#participantEntrypoints.includes(callback)) {\n      throw new FunctionExistsError('entrypoints cannot be added more than once');\n    }\n\n    this.#participantEntrypoints.push(callback);\n  }\n}\n\nexport class JobProcess {\n  #pid = process.pid;\n  userData: { [id: string]: unknown } = {};\n\n  get pid(): number {\n    return this.#pid;\n  }\n}\n\n/**\n * A request sent by the server to spawn a new agent job.\n *\n * @remarks\n * For most applications, this is best left to the default, which simply accepts the job and\n * handles the logic inside the entrypoint function. This class is useful for vetting which\n * requests should fill idle processes and which should be outright rejected.\n */\nexport class JobRequest {\n  #job: proto.Job;\n  #onReject: () => Promise<void>;\n  #onAccept: (args: JobAcceptArguments) => Promise<void>;\n\n  /** @internal */\n  constructor(\n    job: proto.Job,\n    onReject: () => Promise<void>,\n    onAccept: (args: JobAcceptArguments) => Promise<void>,\n  ) {\n    this.#job = job;\n    this.#onReject = onReject;\n    this.#onAccept = onAccept;\n  }\n\n  /** @returns The ID of the job, set by the LiveKit server */\n  get id(): string {\n    return this.#job.id;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get job(): proto.Job {\n    return this.#job;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get room(): proto.Room | undefined {\n    return this.#job.room;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get publisher(): proto.ParticipantInfo | undefined {\n    return this.#job.participant;\n  }\n\n  /** @returns The agent's name, as set in {@link WorkerOptions} */\n  get agentName(): string {\n    return this.#job.agentName;\n  }\n\n  /** Rejects the job. */\n  async reject() {\n    await this.#onReject();\n  }\n\n  /** Accepts the job, launching it on an idle child process. */\n  async accept(name = '', identity = '', metadata = '', attributes?: { [key: string]: string }) {\n    if (identity === '') identity = 'agent-' + this.id;\n\n    this.#onAccept({ name, identity, metadata, attributes });\n  }\n}\n"],"mappings":"AAWA,SAAS,iBAAiB,WAAW,iBAAiB;AAEtD,SAAS,WAAW;AAGb,IAAK,gBAAL,kBAAKA,mBAAL;AACL,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AAJU,SAAAA;AAAA,GAAA;AAsBL,MAAM,4BAA4B,MAAM;AAAA,EAC7C,YAAY,KAAc;AACxB,UAAM,GAAG;AACT,WAAO,eAAe,MAAM,WAAW,SAAS;AAAA,EAClD;AACF;AAGO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA,oBAA6C,CAAC;AAAA,EAC9C,0BAAwF,CAAC;AAAA,EACzF,oBAKI,CAAC;AAAA,EACL;AAAA,EAEA,YACE,MACA,MACA,MACA,WACA,YACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,SAAK,yBAAyB,KAAK,uBAAuB,KAAK,IAAI;AACnE,SAAK,MAAM,GAAG,UAAU,sBAAsB,KAAK,sBAAsB;AACzE,SAAK,UAAU,IAAI,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,QAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,oBAAoB,UAA+B;AACjD,SAAK,kBAAkB,KAAK,QAAQ;AAAA,EACtC;AAAA,EAEA,MAAM,mBAAmB,UAA+C;AACtE,QAAI,CAAC,KAAK,MAAM,aAAa;AAC3B,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AAEA,eAAW,KAAK,KAAK,MAAM,mBAAmB,OAAO,GAAG;AACtD,WAAK,CAAC,YAAY,EAAE,aAAa,aAAa,EAAE,KAAK,QAAQ,gBAAgB,OAAO;AAClF,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,yBAAyB,CAAC,gBAAmC;AACjE,aACG,CAAC,YAAY,YAAY,aAAa,aACvC,YAAY,KAAK,QAAQ,gBAAgB,OACzC;AACA,wBAAc;AACd,kBAAQ,WAAW;AAAA,QACrB;AAAA,MACF;AACA,YAAM,iBAAiB,MAAM;AAC3B,sBAAc;AACd,eAAO,IAAI,MAAM,iDAAiD,CAAC;AAAA,MACrE;AAEA,YAAM,gBAAgB,MAAM;AAC1B,aAAK,MAAM,IAAI,UAAU,sBAAsB,sBAAsB;AACrE,aAAK,MAAM,IAAI,UAAU,cAAc,cAAc;AAAA,MACvD;AAEA,WAAK,MAAM,GAAG,UAAU,sBAAsB,sBAAsB;AACpE,WAAK,MAAM,GAAG,UAAU,cAAc,cAAc;AAAA,IACtD,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,MACA,gBAA+B,uBAC/B,WACA;AACA,UAAM,OAAO;AAAA,MACX;AAAA,MACA,eAAe,iBAAiB;AAAA,MAChC;AAAA,MACA,UAAU;AAAA,IACZ;AAEA,UAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,MAAM,OAAO,IAAI;AAC/D,SAAK,WAAW;AAEhB,SAAK,MAAM,mBAAmB,QAAQ,KAAK,sBAAsB;AAEjE,QAAI,CAAC,oBAA0B,kBAAwB,EAAE,SAAS,aAAa,GAAG;AAChF,WAAK,MAAM,mBAAmB,QAAQ,CAAC,MAAM;AAC3C,UAAE,kBAAkB,QAAQ,CAAC,QAAQ;AACnC,cACG,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,cACrE,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,YACtE;AACA,gBAAI,cAAc,IAAI;AAAA,UACxB;AAAA,QACF,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,IAAI;AACpB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA;AAAA,EAGA,uBAAuB,GAAsB;AA9L/C;AA+LI,eAAW,YAAY,KAAK,yBAAyB;AACnD,YAAI,UAAK,kBAAkB,EAAE,QAAS,MAAlC,mBAAqC,aAAY,UAAU;AAC7D,aAAK,QAAQ;AAAA,UACX;AAAA,UACA,EAAE;AAAA,QACJ;AAAA,MACF;AACA,YAAM,SAAS,SAAS,MAAM,CAAC;AAC/B,aAAO,QAAQ,MAAM,OAAO,KAAK,kBAAkB,EAAE,QAAS,CAAC;AAC/D,WAAK,kBAAkB,EAAE,QAAS,IAAI,EAAE,UAAU,OAAO;AAAA,IAC3D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,yBAAyB,UAAoE;AAC3F,QAAI,KAAK,wBAAwB,SAAS,QAAQ,GAAG;AACnD,YAAM,IAAI,oBAAoB,4CAA4C;AAAA,IAC5E;AAEA,SAAK,wBAAwB,KAAK,QAAQ;AAAA,EAC5C;AACF;AAEO,MAAM,WAAW;AAAA,EACtB,OAAO,QAAQ;AAAA,EACf,WAAsC,CAAC;AAAA,EAEvC,IAAI,MAAc;AAChB,WAAO,KAAK;AAAA,EACd;AACF;AAUO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,YACE,KACA,UACA,UACA;AACA,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,KAAa;AACf,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,MAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,OAA+B;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAA+C;AACjD,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB;AACtB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,MAAM,SAAS;AACb,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAO,IAAI,WAAW,IAAI,WAAW,IAAI,YAAwC;AAC5F,QAAI,aAAa,GAAI,YAAW,WAAW,KAAK;AAEhD,SAAK,UAAU,EAAE,MAAM,UAAU,UAAU,WAAW,CAAC;AAAA,EACzD;AACF;","names":["AutoSubscribe"]}
+{"version":3,"sources":["../src/job.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as proto from '@livekit/protocol';\nimport type {\n  E2EEOptions,\n  LocalParticipant,\n  RemoteParticipant,\n  Room,\n  RtcConfiguration,\n} from '@livekit/rtc-node';\nimport { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';\nimport type { Logger } from 'pino';\nimport type { InferenceExecutor } from './ipc/inference_executor.js';\nimport { log } from './log.js';\n\nexport class CurrentJobContext {\n  static #current: JobContext;\n\n  constructor(proc: JobContext) {\n    CurrentJobContext.#current = proc;\n  }\n\n  static getCurrent(): JobContext {\n    return CurrentJobContext.#current;\n  }\n}\n\n/** Which tracks, if any, should the agent automatically subscribe to? */\nexport enum AutoSubscribe {\n  SUBSCRIBE_ALL,\n  SUBSCRIBE_NONE,\n  VIDEO_ONLY,\n  AUDIO_ONLY,\n}\n\nexport type JobAcceptArguments = {\n  name: string;\n  identity: string;\n  metadata: string;\n  attributes?: { [key: string]: string };\n};\n\nexport type RunningJobInfo = {\n  acceptArguments: JobAcceptArguments;\n  job: proto.Job;\n  url: string;\n  token: string;\n};\n\n/** Attempted to add a function callback, but the function already exists. */\nexport class FunctionExistsError extends Error {\n  constructor(msg?: string) {\n    super(msg);\n    Object.setPrototypeOf(this, new.target.prototype);\n  }\n}\n\n/** The job and environment context as seen by the agent, accessible by the entrypoint function. */\nexport class JobContext {\n  #proc: JobProcess;\n  #info: RunningJobInfo;\n  #room: Room;\n  #onConnect: () => void;\n  #onShutdown: (s: string) => void;\n  /** @internal */\n  shutdownCallbacks: (() => Promise<void>)[] = [];\n  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];\n  #participantTasks: {\n    [id: string]: {\n      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;\n      result: Promise<void>;\n    };\n  } = {};\n  #logger: Logger;\n  #inferenceExecutor: InferenceExecutor;\n\n  constructor(\n    proc: JobProcess,\n    info: RunningJobInfo,\n    room: Room,\n    onConnect: () => void,\n    onShutdown: (s: string) => void,\n    inferenceExecutor: InferenceExecutor,\n  ) {\n    this.#proc = proc;\n    this.#info = info;\n    this.#room = room;\n    this.#onConnect = onConnect;\n    this.#onShutdown = onShutdown;\n    this.onParticipantConnected = this.onParticipantConnected.bind(this);\n    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);\n    this.#logger = log().child({ info: this.#info });\n    this.#inferenceExecutor = inferenceExecutor;\n  }\n\n  get proc(): JobProcess {\n    return this.#proc;\n  }\n\n  get job(): proto.Job {\n    return this.#info.job;\n  }\n\n  /** @returns The room the agent was called into */\n  get room(): Room {\n    return this.#room;\n  }\n\n  /** @returns The agent's participant if connected to the room, otherwise `undefined` */\n  get agent(): LocalParticipant | undefined {\n    return this.#room.localParticipant;\n  }\n\n  /** @returns The global inference executor */\n  get inferenceExecutor(): InferenceExecutor {\n    return this.#inferenceExecutor;\n  }\n\n  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */\n  addShutdownCallback(callback: () => Promise<void>) {\n    this.shutdownCallbacks.push(callback);\n  }\n\n  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {\n    if (!this.#room.isConnected) {\n      throw new Error('room is not connected');\n    }\n\n    for (const p of this.#room.remoteParticipants.values()) {\n      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {\n        return p;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const onParticipantConnected = (participant: RemoteParticipant) => {\n        if (\n          (!identity || participant.identity === identity) &&\n          participant.info.kind != ParticipantKind.AGENT\n        ) {\n          clearHandlers();\n          resolve(participant);\n        }\n      };\n      const onDisconnected = () => {\n        clearHandlers();\n        reject(new Error('Room disconnected while waiting for participant'));\n      };\n\n      const clearHandlers = () => {\n        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);\n        this.#room.off(RoomEvent.Disconnected, onDisconnected);\n      };\n\n      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);\n      this.#room.on(RoomEvent.Disconnected, onDisconnected);\n    });\n  }\n\n  /**\n   * Connects the agent to the room.\n   *\n   * @remarks\n   * It is recommended to run this command as early in the function as possible, as executing it\n   * later may cause noticeable delay between user and agent joins.\n   *\n   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |\n   * @livekit/rtc-node} for more information about the parameters.\n   */\n  async connect(\n    e2ee?: E2EEOptions,\n    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,\n    rtcConfig?: RtcConfiguration,\n  ) {\n    const opts = {\n      e2ee,\n      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,\n      rtcConfig,\n      dynacast: false,\n    };\n\n    await this.#room.connect(this.#info.url, this.#info.token, opts);\n    this.#onConnect();\n\n    this.#room.remoteParticipants.forEach(this.onParticipantConnected);\n\n    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {\n      this.#room.remoteParticipants.forEach((p) => {\n        p.trackPublications.forEach((pub) => {\n          if (\n            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||\n            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)\n          ) {\n            pub.setSubscribed(true);\n          }\n        });\n      });\n    }\n  }\n\n  /**\n   * Gracefully shuts down the job, and runs all shutdown promises.\n   *\n   * @param reason - Optional reason for shutdown\n   */\n  shutdown(reason = '') {\n    this.#onShutdown(reason);\n  }\n\n  /** @internal */\n  onParticipantConnected(p: RemoteParticipant) {\n    for (const callback of this.#participantEntrypoints) {\n      if (this.#participantTasks[p.identity!]?.callback == callback) {\n        this.#logger.warn(\n          'a participant has joined before a prior prticipant task matching the same identity has finished:',\n          p.identity,\n        );\n      }\n      const result = callback(this, p);\n      result.finally(() => delete this.#participantTasks[p.identity!]);\n      this.#participantTasks[p.identity!] = { callback, result };\n    }\n  }\n\n  /**\n   * Adds a promise to be awaited whenever a new participant joins the room.\n   *\n   * @throws {@link FunctionExistsError} if an entrypoint already exists\n   */\n  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {\n    if (this.#participantEntrypoints.includes(callback)) {\n      throw new FunctionExistsError('entrypoints cannot be added more than once');\n    }\n\n    this.#participantEntrypoints.push(callback);\n  }\n}\n\nexport class JobProcess {\n  #pid = process.pid;\n  userData: { [id: string]: unknown } = {};\n\n  get pid(): number {\n    return this.#pid;\n  }\n}\n\n/**\n * A request sent by the server to spawn a new agent job.\n *\n * @remarks\n * For most applications, this is best left to the default, which simply accepts the job and\n * handles the logic inside the entrypoint function. This class is useful for vetting which\n * requests should fill idle processes and which should be outright rejected.\n */\nexport class JobRequest {\n  #job: proto.Job;\n  #onReject: () => Promise<void>;\n  #onAccept: (args: JobAcceptArguments) => Promise<void>;\n\n  /** @internal */\n  constructor(\n    job: proto.Job,\n    onReject: () => Promise<void>,\n    onAccept: (args: JobAcceptArguments) => Promise<void>,\n  ) {\n    this.#job = job;\n    this.#onReject = onReject;\n    this.#onAccept = onAccept;\n  }\n\n  /** @returns The ID of the job, set by the LiveKit server */\n  get id(): string {\n    return this.#job.id;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get job(): proto.Job {\n    return this.#job;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get room(): proto.Room | undefined {\n    return this.#job.room;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get publisher(): proto.ParticipantInfo | undefined {\n    return this.#job.participant;\n  }\n\n  /** @returns The agent's name, as set in {@link WorkerOptions} */\n  get agentName(): string {\n    return this.#job.agentName;\n  }\n\n  /** Rejects the job. */\n  async reject() {\n    await this.#onReject();\n  }\n\n  /** Accepts the job, launching it on an idle child process. */\n  async accept(name = '', identity = '', metadata = '', attributes?: { [key: string]: string }) {\n    if (identity === '') identity = 'agent-' + this.id;\n\n    this.#onAccept({ name, identity, metadata, attributes });\n  }\n}\n"],"mappings":"AAWA,SAAS,iBAAiB,WAAW,iBAAiB;AAGtD,SAAS,WAAW;AAEb,MAAM,kBAAkB;AAAA,EAC7B,OAAO;AAAA,EAEP,YAAY,MAAkB;AAC5B,sBAAkB,WAAW;AAAA,EAC/B;AAAA,EAEA,OAAO,aAAyB;AAC9B,WAAO,kBAAkB;AAAA,EAC3B;AACF;AAGO,IAAK,gBAAL,kBAAKA,mBAAL;AACL,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AAJU,SAAAA;AAAA,GAAA;AAsBL,MAAM,4BAA4B,MAAM;AAAA,EAC7C,YAAY,KAAc;AACxB,UAAM,GAAG;AACT,WAAO,eAAe,MAAM,WAAW,SAAS;AAAA,EAClD;AACF;AAGO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA,oBAA6C,CAAC;AAAA,EAC9C,0BAAwF,CAAC;AAAA,EACzF,oBAKI,CAAC;AAAA,EACL;AAAA,EACA;AAAA,EAEA,YACE,MACA,MACA,MACA,WACA,YACA,mBACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,SAAK,yBAAyB,KAAK,uBAAuB,KAAK,IAAI;AACnE,SAAK,MAAM,GAAG,UAAU,sBAAsB,KAAK,sBAAsB;AACzE,SAAK,UAAU,IAAI,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAC/C,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,QAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,oBAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,oBAAoB,UAA+B;AACjD,SAAK,kBAAkB,KAAK,QAAQ;AAAA,EACtC;AAAA,EAEA,MAAM,mBAAmB,UAA+C;AACtE,QAAI,CAAC,KAAK,MAAM,aAAa;AAC3B,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AAEA,eAAW,KAAK,KAAK,MAAM,mBAAmB,OAAO,GAAG;AACtD,WAAK,CAAC,YAAY,EAAE,aAAa,aAAa,EAAE,KAAK,QAAQ,gBAAgB,OAAO;AAClF,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,yBAAyB,CAAC,gBAAmC;AACjE,aACG,CAAC,YAAY,YAAY,aAAa,aACvC,YAAY,KAAK,QAAQ,gBAAgB,OACzC;AACA,wBAAc;AACd,kBAAQ,WAAW;AAAA,QACrB;AAAA,MACF;AACA,YAAM,iBAAiB,MAAM;AAC3B,sBAAc;AACd,eAAO,IAAI,MAAM,iDAAiD,CAAC;AAAA,MACrE;AAEA,YAAM,gBAAgB,MAAM;AAC1B,aAAK,MAAM,IAAI,UAAU,sBAAsB,sBAAsB;AACrE,aAAK,MAAM,IAAI,UAAU,cAAc,cAAc;AAAA,MACvD;AAEA,WAAK,MAAM,GAAG,UAAU,sBAAsB,sBAAsB;AACpE,WAAK,MAAM,GAAG,UAAU,cAAc,cAAc;AAAA,IACtD,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,MACA,gBAA+B,uBAC/B,WACA;AACA,UAAM,OAAO;AAAA,MACX;AAAA,MACA,eAAe,iBAAiB;AAAA,MAChC;AAAA,MACA,UAAU;AAAA,IACZ;AAEA,UAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,MAAM,OAAO,IAAI;AAC/D,SAAK,WAAW;AAEhB,SAAK,MAAM,mBAAmB,QAAQ,KAAK,sBAAsB;AAEjE,QAAI,CAAC,oBAA0B,kBAAwB,EAAE,SAAS,aAAa,GAAG;AAChF,WAAK,MAAM,mBAAmB,QAAQ,CAAC,MAAM;AAC3C,UAAE,kBAAkB,QAAQ,CAAC,QAAQ;AACnC,cACG,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,cACrE,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,YACtE;AACA,gBAAI,cAAc,IAAI;AAAA,UACxB;AAAA,QACF,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,IAAI;AACpB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA;AAAA,EAGA,uBAAuB,GAAsB;AAnN/C;AAoNI,eAAW,YAAY,KAAK,yBAAyB;AACnD,YAAI,UAAK,kBAAkB,EAAE,QAAS,MAAlC,mBAAqC,aAAY,UAAU;AAC7D,aAAK,QAAQ;AAAA,UACX;AAAA,UACA,EAAE;AAAA,QACJ;AAAA,MACF;AACA,YAAM,SAAS,SAAS,MAAM,CAAC;AAC/B,aAAO,QAAQ,MAAM,OAAO,KAAK,kBAAkB,EAAE,QAAS,CAAC;AAC/D,WAAK,kBAAkB,EAAE,QAAS,IAAI,EAAE,UAAU,OAAO;AAAA,IAC3D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,yBAAyB,UAAoE;AAC3F,QAAI,KAAK,wBAAwB,SAAS,QAAQ,GAAG;AACnD,YAAM,IAAI,oBAAoB,4CAA4C;AAAA,IAC5E;AAEA,SAAK,wBAAwB,KAAK,QAAQ;AAAA,EAC5C;AACF;AAEO,MAAM,WAAW;AAAA,EACtB,OAAO,QAAQ;AAAA,EACf,WAAsC,CAAC;AAAA,EAEvC,IAAI,MAAc;AAChB,WAAO,KAAK;AAAA,EACd;AACF;AAUO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,YACE,KACA,UACA,UACA;AACA,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,KAAa;AACf,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,MAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,OAA+B;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAA+C;AACjD,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB;AACtB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,MAAM,SAAS;AACb,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAO,IAAI,WAAW,IAAI,WAAW,IAAI,YAAwC;AAC5F,QAAI,aAAa,GAAI,YAAW,WAAW,KAAK;AAEhD,SAAK,UAAU,EAAE,MAAM,UAAU,UAAU,WAAW,CAAC;AAAA,EACzD;AACF;","names":["AutoSubscribe"]}

package/dist/multimodal/agent_playout.cjs

@@ -33,7 +33,7 @@ class PlayoutHandle extends import_node_events.EventEmitter {
   #itemId;
   #contentIndex;
   /** @internal */
-  transcriptionFwd;
+  synchronizer;
   /** @internal */
   doneFut;
   /** @internal */
@@ -45,13 +45,13 @@ class PlayoutHandle extends import_node_events.EventEmitter {
   /** @internal */
   totalPlayedTime;
   // Set when playout is done
-  constructor(audioSource, sampleRate, itemId, contentIndex, transcriptionFwd) {
+  constructor(audioSource, sampleRate, itemId, contentIndex, synchronizer) {
     super();
     this.#audioSource = audioSource;
     this.#sampleRate = sampleRate;
     this.#itemId = itemId;
     this.#contentIndex = contentIndex;
-    this.transcriptionFwd = transcriptionFwd;
+    this.synchronizer = synchronizer;
     this.doneFut = new import_utils.Future();
     this.intFut = new import_utils.Future();
     this.#interrupted = false;
@@ -65,12 +65,15 @@ class PlayoutHandle extends import_node_events.EventEmitter {
     if (this.totalPlayedTime !== void 0) {
       return Math.floor(this.totalPlayedTime * this.#sampleRate);
     }
-    return Math.floor(
-      (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1e3)
+    return Math.max(
+      0,
+      Math.floor(
+        (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1e3)
+      )
     );
   }
   get textChars() {
-    return this.transcriptionFwd.currentCharacterIndex;
+    return this.synchronizer.playedText.length;
   }
   get contentIndex() {
     return this.#contentIndex;
@@ -103,13 +106,13 @@ class AgentPlayout extends import_node_events.EventEmitter {
     this.#inFrameSize = inFrameSize;
     this.#outFrameSize = outFrameSize;
   }
-  play(itemId, contentIndex, transcriptionFwd, textStream, audioStream) {
+  play(itemId, contentIndex, synchronizer, textStream, audioStream) {
     const handle = new PlayoutHandle(
       this.#audioSource,
       this.#sampleRate,
       itemId,
       contentIndex,
-      transcriptionFwd
+      synchronizer
     );
     this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
     return handle;
@@ -137,8 +140,9 @@ class AgentPlayout extends import_node_events.EventEmitter {
                   if (cancelledText || cancelled) {
                     break;
                   }
-                  handle.transcriptionFwd.pushText(text);
+                  handle.synchronizer.pushText(text);
                 }
+                handle.synchronizer.markTextSegmentEnd();
                 resolveText();
               } catch (error) {
                 rejectText(error);
@@ -163,11 +167,11 @@ class AgentPlayout extends import_node_events.EventEmitter {
                     break;
                   }
                   if (firstFrame) {
-                    handle.transcriptionFwd.start();
+                    handle.synchronizer.segmentPlayoutStarted();
                     this.emit("playout_started");
                     firstFrame = false;
                   }
-                  handle.transcriptionFwd.pushAudio(frame);
+                  handle.synchronizer.pushAudio(frame);
                   for (const f of bstream.write(frame.data.buffer)) {
                     handle.pushedDuration += f.samplesPerChannel / f.sampleRate * 1e3;
                     await this.#audioSource.captureFrame(f);
@@ -178,7 +182,7 @@ class AgentPlayout extends import_node_events.EventEmitter {
                     handle.pushedDuration += f.samplesPerChannel / f.sampleRate * 1e3;
                     await this.#audioSource.captureFrame(f);
                   }
-                  handle.transcriptionFwd.markAudioComplete();
+                  handle.synchronizer.markAudioSegmentEnd();
                   await this.#audioSource.waitForPlayout();
                 }
                 resolveCapture();
@@ -197,19 +201,17 @@ class AgentPlayout extends import_node_events.EventEmitter {
             }
             handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
             if (handle.interrupted || captureTask.error) {
+              await handle.synchronizer.close(true);
               this.#audioSource.clearQueue();
             }
             if (!readTextTask.isCancelled) {
               await (0, import_utils.gracefullyCancel)(readTextTask);
             }
             if (!firstFrame) {
-              if (!handle.interrupted) {
-                handle.transcriptionFwd.markTextComplete();
-              }
               this.emit("playout_stopped", handle.interrupted);
             }
             handle.doneFut.resolve();
-            await handle.transcriptionFwd.close(handle.interrupted);
+            await handle.synchronizer.close(false);
           }
           resolve();
         } catch (error) {

package/dist/multimodal/agent_playout.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/multimodal/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { type AudioSource } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport type { TranscriptionForwarder } from '../transcription.js';\nimport { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\n\nexport const proto = {};\n\nexport class PlayoutHandle extends EventEmitter {\n  #audioSource: AudioSource;\n  #sampleRate: number;\n  #itemId: string;\n  #contentIndex: number;\n  /** @internal */\n  transcriptionFwd: TranscriptionForwarder;\n  /** @internal */\n  doneFut: Future;\n  /** @internal */\n  intFut: Future;\n  /** @internal */\n  #interrupted: boolean;\n  /** @internal */\n  pushedDuration: number;\n  /** @internal */\n  totalPlayedTime: number | undefined; // Set when playout is done\n\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    itemId: string,\n    contentIndex: number,\n    transcriptionFwd: TranscriptionForwarder,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#sampleRate = sampleRate;\n    this.#itemId = itemId;\n    this.#contentIndex = contentIndex;\n    this.transcriptionFwd = transcriptionFwd;\n    this.doneFut = new Future();\n    this.intFut = new Future();\n    this.#interrupted = false;\n    this.pushedDuration = 0;\n    this.totalPlayedTime = undefined;\n  }\n\n  get itemId(): string {\n    return this.#itemId;\n  }\n\n  get audioSamples(): number {\n    if (this.totalPlayedTime !== undefined) {\n      return Math.floor(this.totalPlayedTime * this.#sampleRate);\n    }\n\n    return Math.floor(\n      (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),\n    );\n  }\n\n  get textChars(): number {\n    return this.transcriptionFwd.currentCharacterIndex;\n  }\n\n  get contentIndex(): number {\n    return this.#contentIndex;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.doneFut.done) return;\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n}\n\nexport class AgentPlayout extends EventEmitter {\n  #audioSource: AudioSource;\n  #playoutTask: CancellablePromise<void> | null;\n  #sampleRate: number;\n  #numChannels: number;\n  #inFrameSize: number;\n  #outFrameSize: number;\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    numChannels: number,\n    inFrameSize: number,\n    outFrameSize: number,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#playoutTask = null;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n    this.#inFrameSize = inFrameSize;\n    this.#outFrameSize = outFrameSize;\n  }\n\n  play(\n    itemId: string,\n    contentIndex: number,\n    transcriptionFwd: TranscriptionForwarder,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): PlayoutHandle {\n    const handle = new PlayoutHandle(\n      this.#audioSource,\n      this.#sampleRate,\n      itemId,\n      contentIndex,\n      transcriptionFwd,\n    );\n    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);\n    return handle;\n  }\n\n  #makePlayoutTask(\n    oldTask: CancellablePromise<void> | null,\n    handle: PlayoutHandle,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): CancellablePromise<void> {\n    return new CancellablePromise<void>((resolve, reject, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      (async () => {\n        try {\n          if (oldTask) {\n            await gracefullyCancel(oldTask);\n          }\n\n          let firstFrame = true;\n\n          const readText = () =>\n            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {\n              let cancelledText = false;\n              onCancelText(() => {\n                cancelledText = true;\n              });\n\n              (async () => {\n                try {\n                  for await (const text of textStream) {\n                    if (cancelledText || cancelled) {\n                      break;\n                    }\n                    handle.transcriptionFwd.pushText(text);\n                  }\n                  resolveText();\n                } catch (error) {\n                  rejectText(error);\n                }\n              })();\n            });\n\n          const capture = () =>\n            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {\n              let cancelledCapture = false;\n              onCancelCapture(() => {\n                cancelledCapture = true;\n              });\n\n              (async () => {\n                try {\n                  const samplesPerChannel = this.#outFrameSize;\n                  const bstream = new AudioByteStream(\n                    this.#sampleRate,\n                    this.#numChannels,\n                    samplesPerChannel,\n                  );\n\n                  for await (const frame of audioStream) {\n                    if (cancelledCapture || cancelled) {\n                      break;\n                    }\n                    if (firstFrame) {\n                      handle.transcriptionFwd.start();\n                      this.emit('playout_started');\n                      firstFrame = false;\n                    }\n\n                    handle.transcriptionFwd.pushAudio(frame);\n\n                    for (const f of bstream.write(frame.data.buffer)) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n                  }\n\n                  if (!cancelledCapture && !cancelled) {\n                    for (const f of bstream.flush()) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n\n                    handle.transcriptionFwd.markAudioComplete();\n\n                    await this.#audioSource.waitForPlayout();\n                  }\n\n                  resolveCapture();\n                } catch (error) {\n                  rejectCapture(error);\n                }\n              })();\n            });\n\n          const readTextTask = readText();\n          const captureTask = capture();\n\n          try {\n            await Promise.race([captureTask, handle.intFut.await]);\n          } finally {\n            if (!captureTask.isCancelled) {\n              await gracefullyCancel(captureTask);\n            }\n\n            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n            if (handle.interrupted || captureTask.error) {\n              this.#audioSource.clearQueue(); // make sure to remove any queued frames\n            }\n\n            if (!readTextTask.isCancelled) {\n              await gracefullyCancel(readTextTask);\n            }\n\n            if (!firstFrame) {\n              if (!handle.interrupted) {\n                handle.transcriptionFwd.markTextComplete();\n              }\n\n              this.emit('playout_stopped', handle.interrupted);\n            }\n\n            handle.doneFut.resolve();\n            await handle.transcriptionFwd.close(handle.interrupted);\n          }\n\n          resolve();\n        } catch (error) {\n          reject(error);\n        }\n      })();\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,mBAAgC;AAEhC,mBAAsF;AAE/E,MAAM,QAAQ,CAAC;AAEf,MAAM,sBAAsB,gCAAa;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA,YACE,aACA,YACA,QACA,cACA,kBACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,gBAAgB;AACrB,SAAK,mBAAmB;AACxB,SAAK,UAAU,IAAI,oBAAO;AAC1B,SAAK,SAAS,IAAI,oBAAO;AACzB,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,KAAK,MAAM,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3D;AAEA,WAAO,KAAK;AAAA,OACT,KAAK,iBAAiB,KAAK,aAAa,mBAAmB,KAAK,cAAc;AAAA,IACjF;AAAA,EACF;AAAA,EAEA,IAAI,YAAoB;AACtB,WAAO,KAAK,iBAAiB;AAAA,EAC/B;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,QAAQ,KAAM;AACvB,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AACF;AAEO,MAAM,qBAAqB,gCAAa;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YACE,aACA,YACA,aACA,aACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,QACA,cACA,kBACA,YACA,aACe;AACf,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,SAAK,eAAe,KAAK,iBAAiB,KAAK,cAAc,QAAQ,YAAY,WAAW;AAC5F,WAAO;AAAA,EACT;AAAA,EAEA,iBACE,SACA,QACA,YACA,aAC0B;AAC1B,WAAO,IAAI,gCAAyB,CAAC,SAAS,QAAQ,aAAa;AACjE,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,OAAC,YAAY;AACX,YAAI;AACF,cAAI,SAAS;AACX,sBAAM,+BAAiB,OAAO;AAAA,UAChC;AAEA,cAAI,aAAa;AAEjB,gBAAM,WAAW,MACf,IAAI,gCAAyB,CAAC,aAAa,YAAY,iBAAiB;AACtE,gBAAI,gBAAgB;AACpB,yBAAa,MAAM;AACjB,8BAAgB;AAAA,YAClB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,iCAAiB,QAAQ,YAAY;AACnC,sBAAI,iBAAiB,WAAW;AAC9B;AAAA,kBACF;AACA,yBAAO,iBAAiB,SAAS,IAAI;AAAA,gBACvC;AACA,4BAAY;AAAA,cACd,SAAS,OAAO;AACd,2BAAW,KAAK;AAAA,cAClB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,UAAU,MACd,IAAI,gCAAyB,CAAC,gBAAgB,eAAe,oBAAoB;AAC/E,gBAAI,mBAAmB;AACvB,4BAAgB,MAAM;AACpB,iCAAmB;AAAA,YACrB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,sBAAM,oBAAoB,KAAK;AAC/B,sBAAM,UAAU,IAAI;AAAA,kBAClB,KAAK;AAAA,kBACL,KAAK;AAAA,kBACL;AAAA,gBACF;AAEA,iCAAiB,SAAS,aAAa;AACrC,sBAAI,oBAAoB,WAAW;AACjC;AAAA,kBACF;AACA,sBAAI,YAAY;AACd,2BAAO,iBAAiB,MAAM;AAC9B,yBAAK,KAAK,iBAAiB;AAC3B,iCAAa;AAAA,kBACf;AAEA,yBAAO,iBAAiB,UAAU,KAAK;AAEvC,6BAAW,KAAK,QAAQ,MAAM,MAAM,KAAK,MAAM,GAAG;AAChD,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAAA,gBACF;AAEA,oBAAI,CAAC,oBAAoB,CAAC,WAAW;AACnC,6BAAW,KAAK,QAAQ,MAAM,GAAG;AAC/B,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAEA,yBAAO,iBAAiB,kBAAkB;AAE1C,wBAAM,KAAK,aAAa,eAAe;AAAA,gBACzC;AAEA,+BAAe;AAAA,cACjB,SAAS,OAAO;AACd,8BAAc,KAAK;AAAA,cACrB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,eAAe,SAAS;AAC9B,gBAAM,cAAc,QAAQ;AAE5B,cAAI;AACF,kBAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,UACvD,UAAE;AACA,gBAAI,CAAC,YAAY,aAAa;AAC5B,wBAAM,+BAAiB,WAAW;AAAA,YACpC;AAEA,mBAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,gBAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,mBAAK,aAAa,WAAW;AAAA,YAC/B;AAEA,gBAAI,CAAC,aAAa,aAAa;AAC7B,wBAAM,+BAAiB,YAAY;AAAA,YACrC;AAEA,gBAAI,CAAC,YAAY;AACf,kBAAI,CAAC,OAAO,aAAa;AACvB,uBAAO,iBAAiB,iBAAiB;AAAA,cAC3C;AAEA,mBAAK,KAAK,mBAAmB,OAAO,WAAW;AAAA,YACjD;AAEA,mBAAO,QAAQ,QAAQ;AACvB,kBAAM,OAAO,iBAAiB,MAAM,OAAO,WAAW;AAAA,UACxD;AAEA,kBAAQ;AAAA,QACV,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,QACd;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AACF;","names":[]}
+{"version":3,"sources":["../../src/multimodal/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { type AudioSource } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport type { TextAudioSynchronizer } from '../transcription.js';\nimport { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\n\nexport const proto = {};\n\nexport class PlayoutHandle extends EventEmitter {\n  #audioSource: AudioSource;\n  #sampleRate: number;\n  #itemId: string;\n  #contentIndex: number;\n  /** @internal */\n  synchronizer: TextAudioSynchronizer;\n  /** @internal */\n  doneFut: Future;\n  /** @internal */\n  intFut: Future;\n  /** @internal */\n  #interrupted: boolean;\n  /** @internal */\n  pushedDuration: number;\n  /** @internal */\n  totalPlayedTime: number | undefined; // Set when playout is done\n\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    itemId: string,\n    contentIndex: number,\n    synchronizer: TextAudioSynchronizer,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#sampleRate = sampleRate;\n    this.#itemId = itemId;\n    this.#contentIndex = contentIndex;\n    this.synchronizer = synchronizer;\n    this.doneFut = new Future();\n    this.intFut = new Future();\n    this.#interrupted = false;\n    this.pushedDuration = 0;\n    this.totalPlayedTime = undefined;\n  }\n\n  get itemId(): string {\n    return this.#itemId;\n  }\n\n  get audioSamples(): number {\n    if (this.totalPlayedTime !== undefined) {\n      return Math.floor(this.totalPlayedTime * this.#sampleRate);\n    }\n\n    return Math.max(\n      0,\n      Math.floor(\n        (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),\n      ),\n    );\n  }\n\n  get textChars(): number {\n    return this.synchronizer.playedText.length;\n  }\n\n  get contentIndex(): number {\n    return this.#contentIndex;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.doneFut.done) return;\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n}\n\nexport class AgentPlayout extends EventEmitter {\n  #audioSource: AudioSource;\n  #playoutTask: CancellablePromise<void> | null;\n  #sampleRate: number;\n  #numChannels: number;\n  #inFrameSize: number;\n  #outFrameSize: number;\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    numChannels: number,\n    inFrameSize: number,\n    outFrameSize: number,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#playoutTask = null;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n    this.#inFrameSize = inFrameSize;\n    this.#outFrameSize = outFrameSize;\n  }\n\n  play(\n    itemId: string,\n    contentIndex: number,\n    synchronizer: TextAudioSynchronizer,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): PlayoutHandle {\n    const handle = new PlayoutHandle(\n      this.#audioSource,\n      this.#sampleRate,\n      itemId,\n      contentIndex,\n      synchronizer,\n    );\n    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);\n    return handle;\n  }\n\n  #makePlayoutTask(\n    oldTask: CancellablePromise<void> | null,\n    handle: PlayoutHandle,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): CancellablePromise<void> {\n    return new CancellablePromise<void>((resolve, reject, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      (async () => {\n        try {\n          if (oldTask) {\n            await gracefullyCancel(oldTask);\n          }\n\n          let firstFrame = true;\n\n          const readText = () =>\n            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {\n              let cancelledText = false;\n              onCancelText(() => {\n                cancelledText = true;\n              });\n\n              (async () => {\n                try {\n                  for await (const text of textStream) {\n                    if (cancelledText || cancelled) {\n                      break;\n                    }\n                    handle.synchronizer.pushText(text);\n                  }\n                  handle.synchronizer.markTextSegmentEnd();\n                  resolveText();\n                } catch (error) {\n                  rejectText(error);\n                }\n              })();\n            });\n\n          const capture = () =>\n            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {\n              let cancelledCapture = false;\n              onCancelCapture(() => {\n                cancelledCapture = true;\n              });\n\n              (async () => {\n                try {\n                  const samplesPerChannel = this.#outFrameSize;\n                  const bstream = new AudioByteStream(\n                    this.#sampleRate,\n                    this.#numChannels,\n                    samplesPerChannel,\n                  );\n\n                  for await (const frame of audioStream) {\n                    if (cancelledCapture || cancelled) {\n                      break;\n                    }\n                    if (firstFrame) {\n                      handle.synchronizer.segmentPlayoutStarted();\n                      this.emit('playout_started');\n                      firstFrame = false;\n                    }\n\n                    handle.synchronizer.pushAudio(frame);\n\n                    for (const f of bstream.write(frame.data.buffer)) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n                  }\n\n                  if (!cancelledCapture && !cancelled) {\n                    for (const f of bstream.flush()) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n\n                    handle.synchronizer.markAudioSegmentEnd();\n\n                    await this.#audioSource.waitForPlayout();\n                  }\n\n                  resolveCapture();\n                } catch (error) {\n                  rejectCapture(error);\n                }\n              })();\n            });\n\n          const readTextTask = readText();\n          const captureTask = capture();\n\n          try {\n            await Promise.race([captureTask, handle.intFut.await]);\n          } finally {\n            if (!captureTask.isCancelled) {\n              await gracefullyCancel(captureTask);\n            }\n\n            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n            if (handle.interrupted || captureTask.error) {\n              await handle.synchronizer.close(true);\n              this.#audioSource.clearQueue(); // make sure to remove any queued frames\n            }\n\n            if (!readTextTask.isCancelled) {\n              await gracefullyCancel(readTextTask);\n            }\n\n            if (!firstFrame) {\n              this.emit('playout_stopped', handle.interrupted);\n            }\n\n            handle.doneFut.resolve();\n            await handle.synchronizer.close(false);\n          }\n\n          resolve();\n        } catch (error) {\n          reject(error);\n        }\n      })();\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,mBAAgC;AAEhC,mBAAsF;AAE/E,MAAM,QAAQ,CAAC;AAEf,MAAM,sBAAsB,gCAAa;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA,YACE,aACA,YACA,QACA,cACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,gBAAgB;AACrB,SAAK,eAAe;AACpB,SAAK,UAAU,IAAI,oBAAO;AAC1B,SAAK,SAAS,IAAI,oBAAO;AACzB,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,KAAK,MAAM,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3D;AAEA,WAAO,KAAK;AAAA,MACV;AAAA,MACA,KAAK;AAAA,SACF,KAAK,iBAAiB,KAAK,aAAa,mBAAmB,KAAK,cAAc;AAAA,MACjF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,YAAoB;AACtB,WAAO,KAAK,aAAa,WAAW;AAAA,EACtC;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,QAAQ,KAAM;AACvB,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AACF;AAEO,MAAM,qBAAqB,gCAAa;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YACE,aACA,YACA,aACA,aACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,QACA,cACA,cACA,YACA,aACe;AACf,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,SAAK,eAAe,KAAK,iBAAiB,KAAK,cAAc,QAAQ,YAAY,WAAW;AAC5F,WAAO;AAAA,EACT;AAAA,EAEA,iBACE,SACA,QACA,YACA,aAC0B;AAC1B,WAAO,IAAI,gCAAyB,CAAC,SAAS,QAAQ,aAAa;AACjE,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,OAAC,YAAY;AACX,YAAI;AACF,cAAI,SAAS;AACX,sBAAM,+BAAiB,OAAO;AAAA,UAChC;AAEA,cAAI,aAAa;AAEjB,gBAAM,WAAW,MACf,IAAI,gCAAyB,CAAC,aAAa,YAAY,iBAAiB;AACtE,gBAAI,gBAAgB;AACpB,yBAAa,MAAM;AACjB,8BAAgB;AAAA,YAClB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,iCAAiB,QAAQ,YAAY;AACnC,sBAAI,iBAAiB,WAAW;AAC9B;AAAA,kBACF;AACA,yBAAO,aAAa,SAAS,IAAI;AAAA,gBACnC;AACA,uBAAO,aAAa,mBAAmB;AACvC,4BAAY;AAAA,cACd,SAAS,OAAO;AACd,2BAAW,KAAK;AAAA,cAClB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,UAAU,MACd,IAAI,gCAAyB,CAAC,gBAAgB,eAAe,oBAAoB;AAC/E,gBAAI,mBAAmB;AACvB,4BAAgB,MAAM;AACpB,iCAAmB;AAAA,YACrB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,sBAAM,oBAAoB,KAAK;AAC/B,sBAAM,UAAU,IAAI;AAAA,kBAClB,KAAK;AAAA,kBACL,KAAK;AAAA,kBACL;AAAA,gBACF;AAEA,iCAAiB,SAAS,aAAa;AACrC,sBAAI,oBAAoB,WAAW;AACjC;AAAA,kBACF;AACA,sBAAI,YAAY;AACd,2BAAO,aAAa,sBAAsB;AAC1C,yBAAK,KAAK,iBAAiB;AAC3B,iCAAa;AAAA,kBACf;AAEA,yBAAO,aAAa,UAAU,KAAK;AAEnC,6BAAW,KAAK,QAAQ,MAAM,MAAM,KAAK,MAAM,GAAG;AAChD,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAAA,gBACF;AAEA,oBAAI,CAAC,oBAAoB,CAAC,WAAW;AACnC,6BAAW,KAAK,QAAQ,MAAM,GAAG;AAC/B,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAEA,yBAAO,aAAa,oBAAoB;AAExC,wBAAM,KAAK,aAAa,eAAe;AAAA,gBACzC;AAEA,+BAAe;AAAA,cACjB,SAAS,OAAO;AACd,8BAAc,KAAK;AAAA,cACrB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,eAAe,SAAS;AAC9B,gBAAM,cAAc,QAAQ;AAE5B,cAAI;AACF,kBAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,UACvD,UAAE;AACA,gBAAI,CAAC,YAAY,aAAa;AAC5B,wBAAM,+BAAiB,WAAW;AAAA,YACpC;AAEA,mBAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,gBAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,oBAAM,OAAO,aAAa,MAAM,IAAI;AACpC,mBAAK,aAAa,WAAW;AAAA,YAC/B;AAEA,gBAAI,CAAC,aAAa,aAAa;AAC7B,wBAAM,+BAAiB,YAAY;AAAA,YACrC;AAEA,gBAAI,CAAC,YAAY;AACf,mBAAK,KAAK,mBAAmB,OAAO,WAAW;AAAA,YACjD;AAEA,mBAAO,QAAQ,QAAQ;AACvB,kBAAM,OAAO,aAAa,MAAM,KAAK;AAAA,UACvC;AAEA,kBAAQ;AAAA,QACV,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,QACd;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/multimodal/agent_playout.d.ts

@@ -2,13 +2,13 @@
 import type { AudioFrame } from '@livekit/rtc-node';
 import { type AudioSource } from '@livekit/rtc-node';
 import { EventEmitter } from 'node:events';
-import type { TranscriptionForwarder } from '../transcription.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { type AsyncIterableQueue, Future } from '../utils.js';
 export declare const proto: {};
 export declare class PlayoutHandle extends EventEmitter {
     #private;
     /** @internal */
-    transcriptionFwd: TranscriptionForwarder;
+    synchronizer: TextAudioSynchronizer;
     /** @internal */
     doneFut: Future;
     /** @internal */
@@ -17,7 +17,7 @@ export declare class PlayoutHandle extends EventEmitter {
     pushedDuration: number;
     /** @internal */
     totalPlayedTime: number | undefined;
-    constructor(audioSource: AudioSource, sampleRate: number, itemId: string, contentIndex: number, transcriptionFwd: TranscriptionForwarder);
+    constructor(audioSource: AudioSource, sampleRate: number, itemId: string, contentIndex: number, synchronizer: TextAudioSynchronizer);
     get itemId(): string;
     get audioSamples(): number;
     get textChars(): number;
@@ -29,6 +29,6 @@ export declare class PlayoutHandle extends EventEmitter {
 export declare class AgentPlayout extends EventEmitter {
     #private;
     constructor(audioSource: AudioSource, sampleRate: number, numChannels: number, inFrameSize: number, outFrameSize: number);
-    play(itemId: string, contentIndex: number, transcriptionFwd: TranscriptionForwarder, textStream: AsyncIterableQueue<string>, audioStream: AsyncIterableQueue<AudioFrame>): PlayoutHandle;
+    play(itemId: string, contentIndex: number, synchronizer: TextAudioSynchronizer, textStream: AsyncIterableQueue<string>, audioStream: AsyncIterableQueue<AudioFrame>): PlayoutHandle;
 }
 //# sourceMappingURL=agent_playout.d.ts.map

package/dist/multimodal/agent_playout.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_playout.d.ts","sourceRoot":"","sources":["../../src/multimodal/agent_playout.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,KAAK,kBAAkB,EAAsB,MAAM,EAAoB,MAAM,aAAa,CAAC;AAEpG,eAAO,MAAM,KAAK,IAAK,CAAC;AAExB,qBAAa,aAAc,SAAQ,YAAY;;IAK7C,gBAAgB;IAChB,gBAAgB,EAAE,sBAAsB,CAAC;IACzC,gBAAgB;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB;IAChB,MAAM,EAAE,MAAM,CAAC;IAGf,gBAAgB;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB;IAChB,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;gBAGlC,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,sBAAsB;IAe1C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,YAAY,IAAI,MAAM,CAQzB;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,SAAS;CAKV;AAED,qBAAa,YAAa,SAAQ,YAAY;;gBAQ1C,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM;IAWtB,IAAI,CACF,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,sBAAsB,EACxC,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,EACtC,WAAW,EAAE,kBAAkB,CAAC,UAAU,CAAC,GAC1C,aAAa;CAiJjB"}
+{"version":3,"file":"agent_playout.d.ts","sourceRoot":"","sources":["../../src/multimodal/agent_playout.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,KAAK,kBAAkB,EAAsB,MAAM,EAAoB,MAAM,aAAa,CAAC;AAEpG,eAAO,MAAM,KAAK,IAAK,CAAC;AAExB,qBAAa,aAAc,SAAQ,YAAY;;IAK7C,gBAAgB;IAChB,YAAY,EAAE,qBAAqB,CAAC;IACpC,gBAAgB;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB;IAChB,MAAM,EAAE,MAAM,CAAC;IAGf,gBAAgB;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB;IAChB,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;gBAGlC,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,qBAAqB;IAerC,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,YAAY,IAAI,MAAM,CAWzB;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,SAAS;CAKV;AAED,qBAAa,YAAa,SAAQ,YAAY;;gBAQ1C,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM;IAWtB,IAAI,CACF,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,qBAAqB,EACnC,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,EACtC,WAAW,EAAE,kBAAkB,CAAC,UAAU,CAAC,GAC1C,aAAa;CA+IjB"}

package/dist/multimodal/agent_playout.js

@@ -8,7 +8,7 @@ class PlayoutHandle extends EventEmitter {
   #itemId;
   #contentIndex;
   /** @internal */
-  transcriptionFwd;
+  synchronizer;
   /** @internal */
   doneFut;
   /** @internal */
@@ -20,13 +20,13 @@ class PlayoutHandle extends EventEmitter {
   /** @internal */
   totalPlayedTime;
   // Set when playout is done
-  constructor(audioSource, sampleRate, itemId, contentIndex, transcriptionFwd) {
+  constructor(audioSource, sampleRate, itemId, contentIndex, synchronizer) {
     super();
     this.#audioSource = audioSource;
     this.#sampleRate = sampleRate;
     this.#itemId = itemId;
     this.#contentIndex = contentIndex;
-    this.transcriptionFwd = transcriptionFwd;
+    this.synchronizer = synchronizer;
     this.doneFut = new Future();
     this.intFut = new Future();
     this.#interrupted = false;
@@ -40,12 +40,15 @@ class PlayoutHandle extends EventEmitter {
     if (this.totalPlayedTime !== void 0) {
       return Math.floor(this.totalPlayedTime * this.#sampleRate);
     }
-    return Math.floor(
-      (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1e3)
+    return Math.max(
+      0,
+      Math.floor(
+        (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1e3)
+      )
     );
   }
   get textChars() {
-    return this.transcriptionFwd.currentCharacterIndex;
+    return this.synchronizer.playedText.length;
   }
   get contentIndex() {
     return this.#contentIndex;
@@ -78,13 +81,13 @@ class AgentPlayout extends EventEmitter {
     this.#inFrameSize = inFrameSize;
     this.#outFrameSize = outFrameSize;
   }
-  play(itemId, contentIndex, transcriptionFwd, textStream, audioStream) {
+  play(itemId, contentIndex, synchronizer, textStream, audioStream) {
     const handle = new PlayoutHandle(
       this.#audioSource,
       this.#sampleRate,
       itemId,
       contentIndex,
-      transcriptionFwd
+      synchronizer
     );
     this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
     return handle;
@@ -112,8 +115,9 @@ class AgentPlayout extends EventEmitter {
                   if (cancelledText || cancelled) {
                     break;
                   }
-                  handle.transcriptionFwd.pushText(text);
+                  handle.synchronizer.pushText(text);
                 }
+                handle.synchronizer.markTextSegmentEnd();
                 resolveText();
               } catch (error) {
                 rejectText(error);
@@ -138,11 +142,11 @@ class AgentPlayout extends EventEmitter {
                     break;
                   }
                   if (firstFrame) {
-                    handle.transcriptionFwd.start();
+                    handle.synchronizer.segmentPlayoutStarted();
                     this.emit("playout_started");
                     firstFrame = false;
                   }
-                  handle.transcriptionFwd.pushAudio(frame);
+                  handle.synchronizer.pushAudio(frame);
                   for (const f of bstream.write(frame.data.buffer)) {
                     handle.pushedDuration += f.samplesPerChannel / f.sampleRate * 1e3;
                     await this.#audioSource.captureFrame(f);
@@ -153,7 +157,7 @@ class AgentPlayout extends EventEmitter {
                     handle.pushedDuration += f.samplesPerChannel / f.sampleRate * 1e3;
                     await this.#audioSource.captureFrame(f);
                   }
-                  handle.transcriptionFwd.markAudioComplete();
+                  handle.synchronizer.markAudioSegmentEnd();
                   await this.#audioSource.waitForPlayout();
                 }
                 resolveCapture();
@@ -172,19 +176,17 @@ class AgentPlayout extends EventEmitter {
             }
             handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
             if (handle.interrupted || captureTask.error) {
+              await handle.synchronizer.close(true);
               this.#audioSource.clearQueue();
             }
             if (!readTextTask.isCancelled) {
               await gracefullyCancel(readTextTask);
             }
             if (!firstFrame) {
-              if (!handle.interrupted) {
-                handle.transcriptionFwd.markTextComplete();
-              }
               this.emit("playout_stopped", handle.interrupted);
             }
             handle.doneFut.resolve();
-            await handle.transcriptionFwd.close(handle.interrupted);
+            await handle.synchronizer.close(false);
           }
           resolve();
         } catch (error) {

package/dist/multimodal/agent_playout.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/multimodal/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { type AudioSource } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport type { TranscriptionForwarder } from '../transcription.js';\nimport { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\n\nexport const proto = {};\n\nexport class PlayoutHandle extends EventEmitter {\n  #audioSource: AudioSource;\n  #sampleRate: number;\n  #itemId: string;\n  #contentIndex: number;\n  /** @internal */\n  transcriptionFwd: TranscriptionForwarder;\n  /** @internal */\n  doneFut: Future;\n  /** @internal */\n  intFut: Future;\n  /** @internal */\n  #interrupted: boolean;\n  /** @internal */\n  pushedDuration: number;\n  /** @internal */\n  totalPlayedTime: number | undefined; // Set when playout is done\n\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    itemId: string,\n    contentIndex: number,\n    transcriptionFwd: TranscriptionForwarder,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#sampleRate = sampleRate;\n    this.#itemId = itemId;\n    this.#contentIndex = contentIndex;\n    this.transcriptionFwd = transcriptionFwd;\n    this.doneFut = new Future();\n    this.intFut = new Future();\n    this.#interrupted = false;\n    this.pushedDuration = 0;\n    this.totalPlayedTime = undefined;\n  }\n\n  get itemId(): string {\n    return this.#itemId;\n  }\n\n  get audioSamples(): number {\n    if (this.totalPlayedTime !== undefined) {\n      return Math.floor(this.totalPlayedTime * this.#sampleRate);\n    }\n\n    return Math.floor(\n      (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),\n    );\n  }\n\n  get textChars(): number {\n    return this.transcriptionFwd.currentCharacterIndex;\n  }\n\n  get contentIndex(): number {\n    return this.#contentIndex;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.doneFut.done) return;\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n}\n\nexport class AgentPlayout extends EventEmitter {\n  #audioSource: AudioSource;\n  #playoutTask: CancellablePromise<void> | null;\n  #sampleRate: number;\n  #numChannels: number;\n  #inFrameSize: number;\n  #outFrameSize: number;\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    numChannels: number,\n    inFrameSize: number,\n    outFrameSize: number,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#playoutTask = null;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n    this.#inFrameSize = inFrameSize;\n    this.#outFrameSize = outFrameSize;\n  }\n\n  play(\n    itemId: string,\n    contentIndex: number,\n    transcriptionFwd: TranscriptionForwarder,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): PlayoutHandle {\n    const handle = new PlayoutHandle(\n      this.#audioSource,\n      this.#sampleRate,\n      itemId,\n      contentIndex,\n      transcriptionFwd,\n    );\n    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);\n    return handle;\n  }\n\n  #makePlayoutTask(\n    oldTask: CancellablePromise<void> | null,\n    handle: PlayoutHandle,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): CancellablePromise<void> {\n    return new CancellablePromise<void>((resolve, reject, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      (async () => {\n        try {\n          if (oldTask) {\n            await gracefullyCancel(oldTask);\n          }\n\n          let firstFrame = true;\n\n          const readText = () =>\n            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {\n              let cancelledText = false;\n              onCancelText(() => {\n                cancelledText = true;\n              });\n\n              (async () => {\n                try {\n                  for await (const text of textStream) {\n                    if (cancelledText || cancelled) {\n                      break;\n                    }\n                    handle.transcriptionFwd.pushText(text);\n                  }\n                  resolveText();\n                } catch (error) {\n                  rejectText(error);\n                }\n              })();\n            });\n\n          const capture = () =>\n            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {\n              let cancelledCapture = false;\n              onCancelCapture(() => {\n                cancelledCapture = true;\n              });\n\n              (async () => {\n                try {\n                  const samplesPerChannel = this.#outFrameSize;\n                  const bstream = new AudioByteStream(\n                    this.#sampleRate,\n                    this.#numChannels,\n                    samplesPerChannel,\n                  );\n\n                  for await (const frame of audioStream) {\n                    if (cancelledCapture || cancelled) {\n                      break;\n                    }\n                    if (firstFrame) {\n                      handle.transcriptionFwd.start();\n                      this.emit('playout_started');\n                      firstFrame = false;\n                    }\n\n                    handle.transcriptionFwd.pushAudio(frame);\n\n                    for (const f of bstream.write(frame.data.buffer)) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n                  }\n\n                  if (!cancelledCapture && !cancelled) {\n                    for (const f of bstream.flush()) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n\n                    handle.transcriptionFwd.markAudioComplete();\n\n                    await this.#audioSource.waitForPlayout();\n                  }\n\n                  resolveCapture();\n                } catch (error) {\n                  rejectCapture(error);\n                }\n              })();\n            });\n\n          const readTextTask = readText();\n          const captureTask = capture();\n\n          try {\n            await Promise.race([captureTask, handle.intFut.await]);\n          } finally {\n            if (!captureTask.isCancelled) {\n              await gracefullyCancel(captureTask);\n            }\n\n            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n            if (handle.interrupted || captureTask.error) {\n              this.#audioSource.clearQueue(); // make sure to remove any queued frames\n            }\n\n            if (!readTextTask.isCancelled) {\n              await gracefullyCancel(readTextTask);\n            }\n\n            if (!firstFrame) {\n              if (!handle.interrupted) {\n                handle.transcriptionFwd.markTextComplete();\n              }\n\n              this.emit('playout_stopped', handle.interrupted);\n            }\n\n            handle.doneFut.resolve();\n            await handle.transcriptionFwd.close(handle.interrupted);\n          }\n\n          resolve();\n        } catch (error) {\n          reject(error);\n        }\n      })();\n    });\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,uBAAuB;AAEhC,SAAkC,oBAAoB,QAAQ,wBAAwB;AAE/E,MAAM,QAAQ,CAAC;AAEf,MAAM,sBAAsB,aAAa;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA,YACE,aACA,YACA,QACA,cACA,kBACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,gBAAgB;AACrB,SAAK,mBAAmB;AACxB,SAAK,UAAU,IAAI,OAAO;AAC1B,SAAK,SAAS,IAAI,OAAO;AACzB,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,KAAK,MAAM,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3D;AAEA,WAAO,KAAK;AAAA,OACT,KAAK,iBAAiB,KAAK,aAAa,mBAAmB,KAAK,cAAc;AAAA,IACjF;AAAA,EACF;AAAA,EAEA,IAAI,YAAoB;AACtB,WAAO,KAAK,iBAAiB;AAAA,EAC/B;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,QAAQ,KAAM;AACvB,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AACF;AAEO,MAAM,qBAAqB,aAAa;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YACE,aACA,YACA,aACA,aACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,QACA,cACA,kBACA,YACA,aACe;AACf,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,SAAK,eAAe,KAAK,iBAAiB,KAAK,cAAc,QAAQ,YAAY,WAAW;AAC5F,WAAO;AAAA,EACT;AAAA,EAEA,iBACE,SACA,QACA,YACA,aAC0B;AAC1B,WAAO,IAAI,mBAAyB,CAAC,SAAS,QAAQ,aAAa;AACjE,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,OAAC,YAAY;AACX,YAAI;AACF,cAAI,SAAS;AACX,kBAAM,iBAAiB,OAAO;AAAA,UAChC;AAEA,cAAI,aAAa;AAEjB,gBAAM,WAAW,MACf,IAAI,mBAAyB,CAAC,aAAa,YAAY,iBAAiB;AACtE,gBAAI,gBAAgB;AACpB,yBAAa,MAAM;AACjB,8BAAgB;AAAA,YAClB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,iCAAiB,QAAQ,YAAY;AACnC,sBAAI,iBAAiB,WAAW;AAC9B;AAAA,kBACF;AACA,yBAAO,iBAAiB,SAAS,IAAI;AAAA,gBACvC;AACA,4BAAY;AAAA,cACd,SAAS,OAAO;AACd,2BAAW,KAAK;AAAA,cAClB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,UAAU,MACd,IAAI,mBAAyB,CAAC,gBAAgB,eAAe,oBAAoB;AAC/E,gBAAI,mBAAmB;AACvB,4BAAgB,MAAM;AACpB,iCAAmB;AAAA,YACrB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,sBAAM,oBAAoB,KAAK;AAC/B,sBAAM,UAAU,IAAI;AAAA,kBAClB,KAAK;AAAA,kBACL,KAAK;AAAA,kBACL;AAAA,gBACF;AAEA,iCAAiB,SAAS,aAAa;AACrC,sBAAI,oBAAoB,WAAW;AACjC;AAAA,kBACF;AACA,sBAAI,YAAY;AACd,2BAAO,iBAAiB,MAAM;AAC9B,yBAAK,KAAK,iBAAiB;AAC3B,iCAAa;AAAA,kBACf;AAEA,yBAAO,iBAAiB,UAAU,KAAK;AAEvC,6BAAW,KAAK,QAAQ,MAAM,MAAM,KAAK,MAAM,GAAG;AAChD,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAAA,gBACF;AAEA,oBAAI,CAAC,oBAAoB,CAAC,WAAW;AACnC,6BAAW,KAAK,QAAQ,MAAM,GAAG;AAC/B,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAEA,yBAAO,iBAAiB,kBAAkB;AAE1C,wBAAM,KAAK,aAAa,eAAe;AAAA,gBACzC;AAEA,+BAAe;AAAA,cACjB,SAAS,OAAO;AACd,8BAAc,KAAK;AAAA,cACrB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,eAAe,SAAS;AAC9B,gBAAM,cAAc,QAAQ;AAE5B,cAAI;AACF,kBAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,UACvD,UAAE;AACA,gBAAI,CAAC,YAAY,aAAa;AAC5B,oBAAM,iBAAiB,WAAW;AAAA,YACpC;AAEA,mBAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,gBAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,mBAAK,aAAa,WAAW;AAAA,YAC/B;AAEA,gBAAI,CAAC,aAAa,aAAa;AAC7B,oBAAM,iBAAiB,YAAY;AAAA,YACrC;AAEA,gBAAI,CAAC,YAAY;AACf,kBAAI,CAAC,OAAO,aAAa;AACvB,uBAAO,iBAAiB,iBAAiB;AAAA,cAC3C;AAEA,mBAAK,KAAK,mBAAmB,OAAO,WAAW;AAAA,YACjD;AAEA,mBAAO,QAAQ,QAAQ;AACvB,kBAAM,OAAO,iBAAiB,MAAM,OAAO,WAAW;AAAA,UACxD;AAEA,kBAAQ;AAAA,QACV,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,QACd;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AACF;","names":[]}
+{"version":3,"sources":["../../src/multimodal/agent_playout.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { type AudioSource } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport type { TextAudioSynchronizer } from '../transcription.js';\nimport { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\n\nexport const proto = {};\n\nexport class PlayoutHandle extends EventEmitter {\n  #audioSource: AudioSource;\n  #sampleRate: number;\n  #itemId: string;\n  #contentIndex: number;\n  /** @internal */\n  synchronizer: TextAudioSynchronizer;\n  /** @internal */\n  doneFut: Future;\n  /** @internal */\n  intFut: Future;\n  /** @internal */\n  #interrupted: boolean;\n  /** @internal */\n  pushedDuration: number;\n  /** @internal */\n  totalPlayedTime: number | undefined; // Set when playout is done\n\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    itemId: string,\n    contentIndex: number,\n    synchronizer: TextAudioSynchronizer,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#sampleRate = sampleRate;\n    this.#itemId = itemId;\n    this.#contentIndex = contentIndex;\n    this.synchronizer = synchronizer;\n    this.doneFut = new Future();\n    this.intFut = new Future();\n    this.#interrupted = false;\n    this.pushedDuration = 0;\n    this.totalPlayedTime = undefined;\n  }\n\n  get itemId(): string {\n    return this.#itemId;\n  }\n\n  get audioSamples(): number {\n    if (this.totalPlayedTime !== undefined) {\n      return Math.floor(this.totalPlayedTime * this.#sampleRate);\n    }\n\n    return Math.max(\n      0,\n      Math.floor(\n        (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),\n      ),\n    );\n  }\n\n  get textChars(): number {\n    return this.synchronizer.playedText.length;\n  }\n\n  get contentIndex(): number {\n    return this.#contentIndex;\n  }\n\n  get interrupted(): boolean {\n    return this.#interrupted;\n  }\n\n  get done(): boolean {\n    return this.doneFut.done || this.#interrupted;\n  }\n\n  interrupt() {\n    if (this.doneFut.done) return;\n    this.intFut.resolve();\n    this.#interrupted = true;\n  }\n}\n\nexport class AgentPlayout extends EventEmitter {\n  #audioSource: AudioSource;\n  #playoutTask: CancellablePromise<void> | null;\n  #sampleRate: number;\n  #numChannels: number;\n  #inFrameSize: number;\n  #outFrameSize: number;\n  constructor(\n    audioSource: AudioSource,\n    sampleRate: number,\n    numChannels: number,\n    inFrameSize: number,\n    outFrameSize: number,\n  ) {\n    super();\n    this.#audioSource = audioSource;\n    this.#playoutTask = null;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n    this.#inFrameSize = inFrameSize;\n    this.#outFrameSize = outFrameSize;\n  }\n\n  play(\n    itemId: string,\n    contentIndex: number,\n    synchronizer: TextAudioSynchronizer,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): PlayoutHandle {\n    const handle = new PlayoutHandle(\n      this.#audioSource,\n      this.#sampleRate,\n      itemId,\n      contentIndex,\n      synchronizer,\n    );\n    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);\n    return handle;\n  }\n\n  #makePlayoutTask(\n    oldTask: CancellablePromise<void> | null,\n    handle: PlayoutHandle,\n    textStream: AsyncIterableQueue<string>,\n    audioStream: AsyncIterableQueue<AudioFrame>,\n  ): CancellablePromise<void> {\n    return new CancellablePromise<void>((resolve, reject, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      (async () => {\n        try {\n          if (oldTask) {\n            await gracefullyCancel(oldTask);\n          }\n\n          let firstFrame = true;\n\n          const readText = () =>\n            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {\n              let cancelledText = false;\n              onCancelText(() => {\n                cancelledText = true;\n              });\n\n              (async () => {\n                try {\n                  for await (const text of textStream) {\n                    if (cancelledText || cancelled) {\n                      break;\n                    }\n                    handle.synchronizer.pushText(text);\n                  }\n                  handle.synchronizer.markTextSegmentEnd();\n                  resolveText();\n                } catch (error) {\n                  rejectText(error);\n                }\n              })();\n            });\n\n          const capture = () =>\n            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {\n              let cancelledCapture = false;\n              onCancelCapture(() => {\n                cancelledCapture = true;\n              });\n\n              (async () => {\n                try {\n                  const samplesPerChannel = this.#outFrameSize;\n                  const bstream = new AudioByteStream(\n                    this.#sampleRate,\n                    this.#numChannels,\n                    samplesPerChannel,\n                  );\n\n                  for await (const frame of audioStream) {\n                    if (cancelledCapture || cancelled) {\n                      break;\n                    }\n                    if (firstFrame) {\n                      handle.synchronizer.segmentPlayoutStarted();\n                      this.emit('playout_started');\n                      firstFrame = false;\n                    }\n\n                    handle.synchronizer.pushAudio(frame);\n\n                    for (const f of bstream.write(frame.data.buffer)) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n                  }\n\n                  if (!cancelledCapture && !cancelled) {\n                    for (const f of bstream.flush()) {\n                      handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;\n                      await this.#audioSource.captureFrame(f);\n                    }\n\n                    handle.synchronizer.markAudioSegmentEnd();\n\n                    await this.#audioSource.waitForPlayout();\n                  }\n\n                  resolveCapture();\n                } catch (error) {\n                  rejectCapture(error);\n                }\n              })();\n            });\n\n          const readTextTask = readText();\n          const captureTask = capture();\n\n          try {\n            await Promise.race([captureTask, handle.intFut.await]);\n          } finally {\n            if (!captureTask.isCancelled) {\n              await gracefullyCancel(captureTask);\n            }\n\n            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;\n\n            if (handle.interrupted || captureTask.error) {\n              await handle.synchronizer.close(true);\n              this.#audioSource.clearQueue(); // make sure to remove any queued frames\n            }\n\n            if (!readTextTask.isCancelled) {\n              await gracefullyCancel(readTextTask);\n            }\n\n            if (!firstFrame) {\n              this.emit('playout_stopped', handle.interrupted);\n            }\n\n            handle.doneFut.resolve();\n            await handle.synchronizer.close(false);\n          }\n\n          resolve();\n        } catch (error) {\n          reject(error);\n        }\n      })();\n    });\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,uBAAuB;AAEhC,SAAkC,oBAAoB,QAAQ,wBAAwB;AAE/E,MAAM,QAAQ,CAAC;AAEf,MAAM,sBAAsB,aAAa;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA,YACE,aACA,YACA,QACA,cACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,gBAAgB;AACrB,SAAK,eAAe;AACpB,SAAK,UAAU,IAAI,OAAO;AAC1B,SAAK,SAAS,IAAI,OAAO;AACzB,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,KAAK,MAAM,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3D;AAEA,WAAO,KAAK;AAAA,MACV;AAAA,MACA,KAAK;AAAA,SACF,KAAK,iBAAiB,KAAK,aAAa,mBAAmB,KAAK,cAAc;AAAA,MACjF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,YAAoB;AACtB,WAAO,KAAK,aAAa,WAAW;AAAA,EACtC;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAgB;AAClB,WAAO,KAAK,QAAQ,QAAQ,KAAK;AAAA,EACnC;AAAA,EAEA,YAAY;AACV,QAAI,KAAK,QAAQ,KAAM;AACvB,SAAK,OAAO,QAAQ;AACpB,SAAK,eAAe;AAAA,EACtB;AACF;AAEO,MAAM,qBAAqB,aAAa;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YACE,aACA,YACA,aACA,aACA,cACA;AACA,UAAM;AACN,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,cAAc;AACnB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,KACE,QACA,cACA,cACA,YACA,aACe;AACf,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,SAAK,eAAe,KAAK,iBAAiB,KAAK,cAAc,QAAQ,YAAY,WAAW;AAC5F,WAAO;AAAA,EACT;AAAA,EAEA,iBACE,SACA,QACA,YACA,aAC0B;AAC1B,WAAO,IAAI,mBAAyB,CAAC,SAAS,QAAQ,aAAa;AACjE,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,OAAC,YAAY;AACX,YAAI;AACF,cAAI,SAAS;AACX,kBAAM,iBAAiB,OAAO;AAAA,UAChC;AAEA,cAAI,aAAa;AAEjB,gBAAM,WAAW,MACf,IAAI,mBAAyB,CAAC,aAAa,YAAY,iBAAiB;AACtE,gBAAI,gBAAgB;AACpB,yBAAa,MAAM;AACjB,8BAAgB;AAAA,YAClB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,iCAAiB,QAAQ,YAAY;AACnC,sBAAI,iBAAiB,WAAW;AAC9B;AAAA,kBACF;AACA,yBAAO,aAAa,SAAS,IAAI;AAAA,gBACnC;AACA,uBAAO,aAAa,mBAAmB;AACvC,4BAAY;AAAA,cACd,SAAS,OAAO;AACd,2BAAW,KAAK;AAAA,cAClB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,UAAU,MACd,IAAI,mBAAyB,CAAC,gBAAgB,eAAe,oBAAoB;AAC/E,gBAAI,mBAAmB;AACvB,4BAAgB,MAAM;AACpB,iCAAmB;AAAA,YACrB,CAAC;AAED,aAAC,YAAY;AACX,kBAAI;AACF,sBAAM,oBAAoB,KAAK;AAC/B,sBAAM,UAAU,IAAI;AAAA,kBAClB,KAAK;AAAA,kBACL,KAAK;AAAA,kBACL;AAAA,gBACF;AAEA,iCAAiB,SAAS,aAAa;AACrC,sBAAI,oBAAoB,WAAW;AACjC;AAAA,kBACF;AACA,sBAAI,YAAY;AACd,2BAAO,aAAa,sBAAsB;AAC1C,yBAAK,KAAK,iBAAiB;AAC3B,iCAAa;AAAA,kBACf;AAEA,yBAAO,aAAa,UAAU,KAAK;AAEnC,6BAAW,KAAK,QAAQ,MAAM,MAAM,KAAK,MAAM,GAAG;AAChD,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAAA,gBACF;AAEA,oBAAI,CAAC,oBAAoB,CAAC,WAAW;AACnC,6BAAW,KAAK,QAAQ,MAAM,GAAG;AAC/B,2BAAO,kBAAmB,EAAE,oBAAoB,EAAE,aAAc;AAChE,0BAAM,KAAK,aAAa,aAAa,CAAC;AAAA,kBACxC;AAEA,yBAAO,aAAa,oBAAoB;AAExC,wBAAM,KAAK,aAAa,eAAe;AAAA,gBACzC;AAEA,+BAAe;AAAA,cACjB,SAAS,OAAO;AACd,8BAAc,KAAK;AAAA,cACrB;AAAA,YACF,GAAG;AAAA,UACL,CAAC;AAEH,gBAAM,eAAe,SAAS;AAC9B,gBAAM,cAAc,QAAQ;AAE5B,cAAI;AACF,kBAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,OAAO,KAAK,CAAC;AAAA,UACvD,UAAE;AACA,gBAAI,CAAC,YAAY,aAAa;AAC5B,oBAAM,iBAAiB,WAAW;AAAA,YACpC;AAEA,mBAAO,kBAAkB,OAAO,iBAAiB,KAAK,aAAa;AAEnE,gBAAI,OAAO,eAAe,YAAY,OAAO;AAC3C,oBAAM,OAAO,aAAa,MAAM,IAAI;AACpC,mBAAK,aAAa,WAAW;AAAA,YAC/B;AAEA,gBAAI,CAAC,aAAa,aAAa;AAC7B,oBAAM,iBAAiB,YAAY;AAAA,YACrC;AAEA,gBAAI,CAAC,YAAY;AACf,mBAAK,KAAK,mBAAmB,OAAO,WAAW;AAAA,YACjD;AAEA,mBAAO,QAAQ,QAAQ;AACvB,kBAAM,OAAO,aAAa,MAAM,KAAK;AAAA,UACvC;AAEA,kBAAQ;AAAA,QACV,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,QACd;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/multimodal/multimodal_agent.cjs

@@ -150,7 +150,7 @@ class MultimodalAgent extends import_node_events.EventEmitter {
         this.emit("agent_stopped_speaking");
         this.#speaking = false;
         if (this.#playingHandle) {
-          let text = this.#playingHandle.transcriptionFwd.text;
+          let text = this.#playingHandle.synchronizer.playedText;
           if (interrupted) {
             text += "\u2026";
           }
@@ -195,16 +195,20 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       this.#session.on("response_content_added", (message) => {
         var _a2;
         if (message.contentType === "text") return;
-        const trFwd = new import_transcription.BasicTranscriptionForwarder(
-          this.room,
-          this.room.localParticipant.identity,
-          this.#getLocalTrackSid(),
-          message.responseId
-        );
+        const synchronizer = new import_transcription.TextAudioSynchronizer(import_transcription.defaultTextSyncOptions);
+        synchronizer.on("textUpdated", (text) => {
+          this.#publishTranscription(
+            this.room.localParticipant.identity,
+            this.#getLocalTrackSid(),
+            text.text,
+            text.final,
+            text.id
+          );
+        });
         const handle = (_a2 = this.#agentPlayout) == null ? void 0 : _a2.play(
           message.itemId,
           message.contentIndex,
-          trFwd,
+          synchronizer,
           message.textStream,
           message.audioStream
         );