@livekit/agents 0.6.0 → 0.6.2

package/dist/job.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/job.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as proto from '@livekit/protocol';\nimport type {\n  E2EEOptions,\n  LocalParticipant,\n  RemoteParticipant,\n  Room,\n  RtcConfiguration,\n} from '@livekit/rtc-node';\nimport { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';\nimport type { Logger } from 'pino';\nimport { log } from './log.js';\n\n/** Which tracks, if any, should the agent automatically subscribe to? */\nexport enum AutoSubscribe {\n  SUBSCRIBE_ALL,\n  SUBSCRIBE_NONE,\n  VIDEO_ONLY,\n  AUDIO_ONLY,\n}\n\nexport type JobAcceptArguments = {\n  name: string;\n  identity: string;\n  metadata: string;\n  attributes?: { [key: string]: string };\n};\n\nexport type RunningJobInfo = {\n  acceptArguments: JobAcceptArguments;\n  job: proto.Job;\n  url: string;\n  token: string;\n};\n\n/** Attempted to add a function callback, but the function already exists. */\nexport class FunctionExistsError extends Error {\n  constructor(msg?: string) {\n    super(msg);\n    Object.setPrototypeOf(this, new.target.prototype);\n  }\n}\n\n/** The job and environment context as seen by the agent, accessible by the entrypoint function. */\nexport class JobContext {\n  #proc: JobProcess;\n  #info: RunningJobInfo;\n  #room: Room;\n  #onConnect: () => void;\n  #onShutdown: (s: string) => void;\n  /** @internal */\n  shutdownCallbacks: (() => Promise<void>)[] = [];\n  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];\n  #participantTasks: {\n    [id: string]: {\n      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;\n      result: Promise<void>;\n    };\n  } = {};\n  #logger: Logger;\n\n  constructor(\n    proc: JobProcess,\n    info: RunningJobInfo,\n    room: Room,\n    onConnect: () => void,\n    onShutdown: (s: string) => void,\n  ) {\n    this.#proc = proc;\n    this.#info = info;\n    this.#room = room;\n    this.#onConnect = onConnect;\n    this.#onShutdown = onShutdown;\n    this.onParticipantConnected = this.onParticipantConnected.bind(this);\n    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);\n    this.#logger = log().child({ info: this.#info });\n  }\n\n  get proc(): JobProcess {\n    return this.#proc;\n  }\n\n  get job(): proto.Job {\n    return this.#info.job;\n  }\n\n  /** @returns The room the agent was called into */\n  get room(): Room {\n    return this.#room;\n  }\n\n  /** @returns The agent's participant if connected to the room, otherwise `undefined` */\n  get agent(): LocalParticipant | undefined {\n    return this.#room.localParticipant;\n  }\n\n  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */\n  addShutdownCallback(callback: () => Promise<void>) {\n    this.shutdownCallbacks.push(callback);\n  }\n\n  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {\n    if (!this.#room.isConnected) {\n      throw new Error('room is not connected');\n    }\n\n    for (const p of this.#room.remoteParticipants.values()) {\n      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {\n        return p;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const onParticipantConnected = (participant: RemoteParticipant) => {\n        if (\n          (!identity || participant.identity === identity) &&\n          participant.info.kind != ParticipantKind.AGENT\n        ) {\n          clearHandlers();\n          resolve(participant);\n        }\n      };\n      const onDisconnected = () => {\n        clearHandlers();\n        reject(new Error('Room disconnected while waiting for participant'));\n      };\n\n      const clearHandlers = () => {\n        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);\n        this.#room.off(RoomEvent.Disconnected, onDisconnected);\n      };\n\n      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);\n      this.#room.on(RoomEvent.Disconnected, onDisconnected);\n    });\n  }\n\n  /**\n   * Connects the agent to the room.\n   *\n   * @remarks\n   * It is recommended to run this command as early in the function as possible, as executing it\n   * later may cause noticeable delay between user and agent joins.\n   *\n   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |\n   * @livekit/rtc-node} for more information about the parameters.\n   */\n  async connect(\n    e2ee?: E2EEOptions,\n    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,\n    rtcConfig?: RtcConfiguration,\n  ) {\n    const opts = {\n      e2ee,\n      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,\n      rtcConfig,\n      dynacast: false,\n    };\n\n    await this.#room.connect(this.#info.url, this.#info.token, opts);\n    this.#onConnect();\n\n    this.#room.remoteParticipants.forEach(this.onParticipantConnected);\n\n    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {\n      this.#room.remoteParticipants.forEach((p) => {\n        p.trackPublications.forEach((pub) => {\n          if (\n            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||\n            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)\n          ) {\n            pub.setSubscribed(true);\n          }\n        });\n      });\n    }\n  }\n\n  /**\n   * Gracefully shuts down the job, and runs all shutdown promises.\n   *\n   * @param reason - Optional reason for shutdown\n   */\n  shutdown(reason = '') {\n    this.#onShutdown(reason);\n  }\n\n  /** @internal */\n  onParticipantConnected(p: RemoteParticipant) {\n    for (const callback of this.#participantEntrypoints) {\n      if (this.#participantTasks[p.identity]?.callback == callback) {\n        this.#logger.warn(\n          'a participant has joined before a prior prticipant task matching the same identity has finished:',\n          p.identity,\n        );\n      }\n      const result = callback(this, p);\n      result.finally(() => delete this.#participantTasks[p.identity]);\n      this.#participantTasks[p.identity] = { callback, result };\n    }\n  }\n\n  /**\n   * Adds a promise to be awaited whenever a new participant joins the room.\n   *\n   * @throws {@link FunctionExistsError} if an entrypoint already exists\n   */\n  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {\n    if (this.#participantEntrypoints.includes(callback)) {\n      throw new FunctionExistsError('entrypoints cannot be added more than once');\n    }\n\n    this.#participantEntrypoints.push(callback);\n  }\n}\n\nexport class JobProcess {\n  #pid = process.pid;\n  userData: { [id: string]: unknown } = {};\n\n  get pid(): number {\n    return this.#pid;\n  }\n}\n\n/**\n * A request sent by the server to spawn a new agent job.\n *\n * @remarks\n * For most applications, this is best left to the default, which simply accepts the job and\n * handles the logic inside the entrypoint function. This class is useful for vetting which\n * requests should fill idle processes and which should be outright rejected.\n */\nexport class JobRequest {\n  #job: proto.Job;\n  #onReject: () => Promise<void>;\n  #onAccept: (args: JobAcceptArguments) => Promise<void>;\n\n  /** @internal */\n  constructor(\n    job: proto.Job,\n    onReject: () => Promise<void>,\n    onAccept: (args: JobAcceptArguments) => Promise<void>,\n  ) {\n    this.#job = job;\n    this.#onReject = onReject;\n    this.#onAccept = onAccept;\n  }\n\n  /** @returns The ID of the job, set by the LiveKit server */\n  get id(): string {\n    return this.#job.id;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get job(): proto.Job {\n    return this.#job;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get room(): proto.Room | undefined {\n    return this.#job.room;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get publisher(): proto.ParticipantInfo | undefined {\n    return this.#job.participant;\n  }\n\n  /** @returns The agent's name, as set in {@link WorkerOptions} */\n  get agentName(): string {\n    return this.#job.agentName;\n  }\n\n  /** Rejects the job. */\n  async reject() {\n    await this.#onReject();\n  }\n\n  /** Accepts the job, launching it on an idle child process. */\n  async accept(name = '', identity = '', metadata = '', attributes?: { [key: string]: string }) {\n    if (identity === '') identity = 'agent-' + this.id;\n\n    this.#onAccept({ name, identity, metadata, attributes });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAAsD;AAEtD,iBAAoB;AAGb,IAAK,gBAAL,kBAAKA,mBAAL;AACL,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AAJU,SAAAA;AAAA,GAAA;AAsBL,MAAM,4BAA4B,MAAM;AAAA,EAC7C,YAAY,KAAc;AACxB,UAAM,GAAG;AACT,WAAO,eAAe,MAAM,WAAW,SAAS;AAAA,EAClD;AACF;AAGO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA,oBAA6C,CAAC;AAAA,EAC9C,0BAAwF,CAAC;AAAA,EACzF,oBAKI,CAAC;AAAA,EACL;AAAA,EAEA,YACE,MACA,MACA,MACA,WACA,YACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,SAAK,yBAAyB,KAAK,uBAAuB,KAAK,IAAI;AACnE,SAAK,MAAM,GAAG,0BAAU,sBAAsB,KAAK,sBAAsB;AACzE,SAAK,cAAU,gBAAI,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,QAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,oBAAoB,UAA+B;AACjD,SAAK,kBAAkB,KAAK,QAAQ;AAAA,EACtC;AAAA,EAEA,MAAM,mBAAmB,UAA+C;AACtE,QAAI,CAAC,KAAK,MAAM,aAAa;AAC3B,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AAEA,eAAW,KAAK,KAAK,MAAM,mBAAmB,OAAO,GAAG;AACtD,WAAK,CAAC,YAAY,EAAE,aAAa,aAAa,EAAE,KAAK,QAAQ,gCAAgB,OAAO;AAClF,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,yBAAyB,CAAC,gBAAmC;AACjE,aACG,CAAC,YAAY,YAAY,aAAa,aACvC,YAAY,KAAK,QAAQ,gCAAgB,OACzC;AACA,wBAAc;AACd,kBAAQ,WAAW;AAAA,QACrB;AAAA,MACF;AACA,YAAM,iBAAiB,MAAM;AAC3B,sBAAc;AACd,eAAO,IAAI,MAAM,iDAAiD,CAAC;AAAA,MACrE;AAEA,YAAM,gBAAgB,MAAM;AAC1B,aAAK,MAAM,IAAI,0BAAU,sBAAsB,sBAAsB;AACrE,aAAK,MAAM,IAAI,0BAAU,cAAc,cAAc;AAAA,MACvD;AAEA,WAAK,MAAM,GAAG,0BAAU,sBAAsB,sBAAsB;AACpE,WAAK,MAAM,GAAG,0BAAU,cAAc,cAAc;AAAA,IACtD,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,MACA,gBAA+B,uBAC/B,WACA;AACA,UAAM,OAAO;AAAA,MACX;AAAA,MACA,eAAe,iBAAiB;AAAA,MAChC;AAAA,MACA,UAAU;AAAA,IACZ;AAEA,UAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,MAAM,OAAO,IAAI;AAC/D,SAAK,WAAW;AAEhB,SAAK,MAAM,mBAAmB,QAAQ,KAAK,sBAAsB;AAEjE,QAAI,CAAC,oBAA0B,kBAAwB,EAAE,SAAS,aAAa,GAAG;AAChF,WAAK,MAAM,mBAAmB,QAAQ,CAAC,MAAM;AAC3C,UAAE,kBAAkB,QAAQ,CAAC,QAAQ;AACnC,cACG,kBAAkB,sBAA4B,IAAI,SAAS,0BAAU,cACrE,kBAAkB,sBAA4B,IAAI,SAAS,0BAAU,YACtE;AACA,gBAAI,cAAc,IAAI;AAAA,UACxB;AAAA,QACF,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,IAAI;AACpB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA;AAAA,EAGA,uBAAuB,GAAsB;AA9L/C;AA+LI,eAAW,YAAY,KAAK,yBAAyB;AACnD,YAAI,UAAK,kBAAkB,EAAE,QAAQ,MAAjC,mBAAoC,aAAY,UAAU;AAC5D,aAAK,QAAQ;AAAA,UACX;AAAA,UACA,EAAE;AAAA,QACJ;AAAA,MACF;AACA,YAAM,SAAS,SAAS,MAAM,CAAC;AAC/B,aAAO,QAAQ,MAAM,OAAO,KAAK,kBAAkB,EAAE,QAAQ,CAAC;AAC9D,WAAK,kBAAkB,EAAE,QAAQ,IAAI,EAAE,UAAU,OAAO;AAAA,IAC1D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,yBAAyB,UAAoE;AAC3F,QAAI,KAAK,wBAAwB,SAAS,QAAQ,GAAG;AACnD,YAAM,IAAI,oBAAoB,4CAA4C;AAAA,IAC5E;AAEA,SAAK,wBAAwB,KAAK,QAAQ;AAAA,EAC5C;AACF;AAEO,MAAM,WAAW;AAAA,EACtB,OAAO,QAAQ;AAAA,EACf,WAAsC,CAAC;AAAA,EAEvC,IAAI,MAAc;AAChB,WAAO,KAAK;AAAA,EACd;AACF;AAUO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,YACE,KACA,UACA,UACA;AACA,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,KAAa;AACf,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,MAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,OAA+B;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAA+C;AACjD,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB;AACtB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,MAAM,SAAS;AACb,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAO,IAAI,WAAW,IAAI,WAAW,IAAI,YAAwC;AAC5F,QAAI,aAAa,GAAI,YAAW,WAAW,KAAK;AAEhD,SAAK,UAAU,EAAE,MAAM,UAAU,UAAU,WAAW,CAAC;AAAA,EACzD;AACF;","names":["AutoSubscribe"]}
+{"version":3,"sources":["../src/job.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as proto from '@livekit/protocol';\nimport type {\n  E2EEOptions,\n  LocalParticipant,\n  RemoteParticipant,\n  Room,\n  RtcConfiguration,\n} from '@livekit/rtc-node';\nimport { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';\nimport type { Logger } from 'pino';\nimport { log } from './log.js';\n\n/** Which tracks, if any, should the agent automatically subscribe to? */\nexport enum AutoSubscribe {\n  SUBSCRIBE_ALL,\n  SUBSCRIBE_NONE,\n  VIDEO_ONLY,\n  AUDIO_ONLY,\n}\n\nexport type JobAcceptArguments = {\n  name: string;\n  identity: string;\n  metadata: string;\n  attributes?: { [key: string]: string };\n};\n\nexport type RunningJobInfo = {\n  acceptArguments: JobAcceptArguments;\n  job: proto.Job;\n  url: string;\n  token: string;\n};\n\n/** Attempted to add a function callback, but the function already exists. */\nexport class FunctionExistsError extends Error {\n  constructor(msg?: string) {\n    super(msg);\n    Object.setPrototypeOf(this, new.target.prototype);\n  }\n}\n\n/** The job and environment context as seen by the agent, accessible by the entrypoint function. */\nexport class JobContext {\n  #proc: JobProcess;\n  #info: RunningJobInfo;\n  #room: Room;\n  #onConnect: () => void;\n  #onShutdown: (s: string) => void;\n  /** @internal */\n  shutdownCallbacks: (() => Promise<void>)[] = [];\n  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];\n  #participantTasks: {\n    [id: string]: {\n      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;\n      result: Promise<void>;\n    };\n  } = {};\n  #logger: Logger;\n\n  constructor(\n    proc: JobProcess,\n    info: RunningJobInfo,\n    room: Room,\n    onConnect: () => void,\n    onShutdown: (s: string) => void,\n  ) {\n    this.#proc = proc;\n    this.#info = info;\n    this.#room = room;\n    this.#onConnect = onConnect;\n    this.#onShutdown = onShutdown;\n    this.onParticipantConnected = this.onParticipantConnected.bind(this);\n    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);\n    this.#logger = log().child({ info: this.#info });\n  }\n\n  get proc(): JobProcess {\n    return this.#proc;\n  }\n\n  get job(): proto.Job {\n    return this.#info.job;\n  }\n\n  /** @returns The room the agent was called into */\n  get room(): Room {\n    return this.#room;\n  }\n\n  /** @returns The agent's participant if connected to the room, otherwise `undefined` */\n  get agent(): LocalParticipant | undefined {\n    return this.#room.localParticipant;\n  }\n\n  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */\n  addShutdownCallback(callback: () => Promise<void>) {\n    this.shutdownCallbacks.push(callback);\n  }\n\n  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {\n    if (!this.#room.isConnected) {\n      throw new Error('room is not connected');\n    }\n\n    for (const p of this.#room.remoteParticipants.values()) {\n      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {\n        return p;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const onParticipantConnected = (participant: RemoteParticipant) => {\n        if (\n          (!identity || participant.identity === identity) &&\n          participant.info.kind != ParticipantKind.AGENT\n        ) {\n          clearHandlers();\n          resolve(participant);\n        }\n      };\n      const onDisconnected = () => {\n        clearHandlers();\n        reject(new Error('Room disconnected while waiting for participant'));\n      };\n\n      const clearHandlers = () => {\n        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);\n        this.#room.off(RoomEvent.Disconnected, onDisconnected);\n      };\n\n      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);\n      this.#room.on(RoomEvent.Disconnected, onDisconnected);\n    });\n  }\n\n  /**\n   * Connects the agent to the room.\n   *\n   * @remarks\n   * It is recommended to run this command as early in the function as possible, as executing it\n   * later may cause noticeable delay between user and agent joins.\n   *\n   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |\n   * @livekit/rtc-node} for more information about the parameters.\n   */\n  async connect(\n    e2ee?: E2EEOptions,\n    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,\n    rtcConfig?: RtcConfiguration,\n  ) {\n    const opts = {\n      e2ee,\n      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,\n      rtcConfig,\n      dynacast: false,\n    };\n\n    await this.#room.connect(this.#info.url, this.#info.token, opts);\n    this.#onConnect();\n\n    this.#room.remoteParticipants.forEach(this.onParticipantConnected);\n\n    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {\n      this.#room.remoteParticipants.forEach((p) => {\n        p.trackPublications.forEach((pub) => {\n          if (\n            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||\n            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)\n          ) {\n            pub.setSubscribed(true);\n          }\n        });\n      });\n    }\n  }\n\n  /**\n   * Gracefully shuts down the job, and runs all shutdown promises.\n   *\n   * @param reason - Optional reason for shutdown\n   */\n  shutdown(reason = '') {\n    this.#onShutdown(reason);\n  }\n\n  /** @internal */\n  onParticipantConnected(p: RemoteParticipant) {\n    for (const callback of this.#participantEntrypoints) {\n      if (this.#participantTasks[p.identity!]?.callback == callback) {\n        this.#logger.warn(\n          'a participant has joined before a prior prticipant task matching the same identity has finished:',\n          p.identity,\n        );\n      }\n      const result = callback(this, p);\n      result.finally(() => delete this.#participantTasks[p.identity!]);\n      this.#participantTasks[p.identity!] = { callback, result };\n    }\n  }\n\n  /**\n   * Adds a promise to be awaited whenever a new participant joins the room.\n   *\n   * @throws {@link FunctionExistsError} if an entrypoint already exists\n   */\n  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {\n    if (this.#participantEntrypoints.includes(callback)) {\n      throw new FunctionExistsError('entrypoints cannot be added more than once');\n    }\n\n    this.#participantEntrypoints.push(callback);\n  }\n}\n\nexport class JobProcess {\n  #pid = process.pid;\n  userData: { [id: string]: unknown } = {};\n\n  get pid(): number {\n    return this.#pid;\n  }\n}\n\n/**\n * A request sent by the server to spawn a new agent job.\n *\n * @remarks\n * For most applications, this is best left to the default, which simply accepts the job and\n * handles the logic inside the entrypoint function. This class is useful for vetting which\n * requests should fill idle processes and which should be outright rejected.\n */\nexport class JobRequest {\n  #job: proto.Job;\n  #onReject: () => Promise<void>;\n  #onAccept: (args: JobAcceptArguments) => Promise<void>;\n\n  /** @internal */\n  constructor(\n    job: proto.Job,\n    onReject: () => Promise<void>,\n    onAccept: (args: JobAcceptArguments) => Promise<void>,\n  ) {\n    this.#job = job;\n    this.#onReject = onReject;\n    this.#onAccept = onAccept;\n  }\n\n  /** @returns The ID of the job, set by the LiveKit server */\n  get id(): string {\n    return this.#job.id;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get job(): proto.Job {\n    return this.#job;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get room(): proto.Room | undefined {\n    return this.#job.room;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get publisher(): proto.ParticipantInfo | undefined {\n    return this.#job.participant;\n  }\n\n  /** @returns The agent's name, as set in {@link WorkerOptions} */\n  get agentName(): string {\n    return this.#job.agentName;\n  }\n\n  /** Rejects the job. */\n  async reject() {\n    await this.#onReject();\n  }\n\n  /** Accepts the job, launching it on an idle child process. */\n  async accept(name = '', identity = '', metadata = '', attributes?: { [key: string]: string }) {\n    if (identity === '') identity = 'agent-' + this.id;\n\n    this.#onAccept({ name, identity, metadata, attributes });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAAsD;AAEtD,iBAAoB;AAGb,IAAK,gBAAL,kBAAKA,mBAAL;AACL,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AAJU,SAAAA;AAAA,GAAA;AAsBL,MAAM,4BAA4B,MAAM;AAAA,EAC7C,YAAY,KAAc;AACxB,UAAM,GAAG;AACT,WAAO,eAAe,MAAM,WAAW,SAAS;AAAA,EAClD;AACF;AAGO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA,oBAA6C,CAAC;AAAA,EAC9C,0BAAwF,CAAC;AAAA,EACzF,oBAKI,CAAC;AAAA,EACL;AAAA,EAEA,YACE,MACA,MACA,MACA,WACA,YACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,SAAK,yBAAyB,KAAK,uBAAuB,KAAK,IAAI;AACnE,SAAK,MAAM,GAAG,0BAAU,sBAAsB,KAAK,sBAAsB;AACzE,SAAK,cAAU,gBAAI,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,QAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,oBAAoB,UAA+B;AACjD,SAAK,kBAAkB,KAAK,QAAQ;AAAA,EACtC;AAAA,EAEA,MAAM,mBAAmB,UAA+C;AACtE,QAAI,CAAC,KAAK,MAAM,aAAa;AAC3B,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AAEA,eAAW,KAAK,KAAK,MAAM,mBAAmB,OAAO,GAAG;AACtD,WAAK,CAAC,YAAY,EAAE,aAAa,aAAa,EAAE,KAAK,QAAQ,gCAAgB,OAAO;AAClF,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,yBAAyB,CAAC,gBAAmC;AACjE,aACG,CAAC,YAAY,YAAY,aAAa,aACvC,YAAY,KAAK,QAAQ,gCAAgB,OACzC;AACA,wBAAc;AACd,kBAAQ,WAAW;AAAA,QACrB;AAAA,MACF;AACA,YAAM,iBAAiB,MAAM;AAC3B,sBAAc;AACd,eAAO,IAAI,MAAM,iDAAiD,CAAC;AAAA,MACrE;AAEA,YAAM,gBAAgB,MAAM;AAC1B,aAAK,MAAM,IAAI,0BAAU,sBAAsB,sBAAsB;AACrE,aAAK,MAAM,IAAI,0BAAU,cAAc,cAAc;AAAA,MACvD;AAEA,WAAK,MAAM,GAAG,0BAAU,sBAAsB,sBAAsB;AACpE,WAAK,MAAM,GAAG,0BAAU,cAAc,cAAc;AAAA,IACtD,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,MACA,gBAA+B,uBAC/B,WACA;AACA,UAAM,OAAO;AAAA,MACX;AAAA,MACA,eAAe,iBAAiB;AAAA,MAChC;AAAA,MACA,UAAU;AAAA,IACZ;AAEA,UAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,MAAM,OAAO,IAAI;AAC/D,SAAK,WAAW;AAEhB,SAAK,MAAM,mBAAmB,QAAQ,KAAK,sBAAsB;AAEjE,QAAI,CAAC,oBAA0B,kBAAwB,EAAE,SAAS,aAAa,GAAG;AAChF,WAAK,MAAM,mBAAmB,QAAQ,CAAC,MAAM;AAC3C,UAAE,kBAAkB,QAAQ,CAAC,QAAQ;AACnC,cACG,kBAAkB,sBAA4B,IAAI,SAAS,0BAAU,cACrE,kBAAkB,sBAA4B,IAAI,SAAS,0BAAU,YACtE;AACA,gBAAI,cAAc,IAAI;AAAA,UACxB;AAAA,QACF,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,IAAI;AACpB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA;AAAA,EAGA,uBAAuB,GAAsB;AA9L/C;AA+LI,eAAW,YAAY,KAAK,yBAAyB;AACnD,YAAI,UAAK,kBAAkB,EAAE,QAAS,MAAlC,mBAAqC,aAAY,UAAU;AAC7D,aAAK,QAAQ;AAAA,UACX;AAAA,UACA,EAAE;AAAA,QACJ;AAAA,MACF;AACA,YAAM,SAAS,SAAS,MAAM,CAAC;AAC/B,aAAO,QAAQ,MAAM,OAAO,KAAK,kBAAkB,EAAE,QAAS,CAAC;AAC/D,WAAK,kBAAkB,EAAE,QAAS,IAAI,EAAE,UAAU,OAAO;AAAA,IAC3D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,yBAAyB,UAAoE;AAC3F,QAAI,KAAK,wBAAwB,SAAS,QAAQ,GAAG;AACnD,YAAM,IAAI,oBAAoB,4CAA4C;AAAA,IAC5E;AAEA,SAAK,wBAAwB,KAAK,QAAQ;AAAA,EAC5C;AACF;AAEO,MAAM,WAAW;AAAA,EACtB,OAAO,QAAQ;AAAA,EACf,WAAsC,CAAC;AAAA,EAEvC,IAAI,MAAc;AAChB,WAAO,KAAK;AAAA,EACd;AACF;AAUO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,YACE,KACA,UACA,UACA;AACA,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,KAAa;AACf,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,MAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,OAA+B;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAA+C;AACjD,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB;AACtB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,MAAM,SAAS;AACb,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAO,IAAI,WAAW,IAAI,WAAW,IAAI,YAAwC;AAC5F,QAAI,aAAa,GAAI,YAAW,WAAW,KAAK;AAEhD,SAAK,UAAU,EAAE,MAAM,UAAU,UAAU,WAAW,CAAC;AAAA,EACzD;AACF;","names":["AutoSubscribe"]}

package/dist/job.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/job.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as proto from '@livekit/protocol';\nimport type {\n  E2EEOptions,\n  LocalParticipant,\n  RemoteParticipant,\n  Room,\n  RtcConfiguration,\n} from '@livekit/rtc-node';\nimport { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';\nimport type { Logger } from 'pino';\nimport { log } from './log.js';\n\n/** Which tracks, if any, should the agent automatically subscribe to? */\nexport enum AutoSubscribe {\n  SUBSCRIBE_ALL,\n  SUBSCRIBE_NONE,\n  VIDEO_ONLY,\n  AUDIO_ONLY,\n}\n\nexport type JobAcceptArguments = {\n  name: string;\n  identity: string;\n  metadata: string;\n  attributes?: { [key: string]: string };\n};\n\nexport type RunningJobInfo = {\n  acceptArguments: JobAcceptArguments;\n  job: proto.Job;\n  url: string;\n  token: string;\n};\n\n/** Attempted to add a function callback, but the function already exists. */\nexport class FunctionExistsError extends Error {\n  constructor(msg?: string) {\n    super(msg);\n    Object.setPrototypeOf(this, new.target.prototype);\n  }\n}\n\n/** The job and environment context as seen by the agent, accessible by the entrypoint function. */\nexport class JobContext {\n  #proc: JobProcess;\n  #info: RunningJobInfo;\n  #room: Room;\n  #onConnect: () => void;\n  #onShutdown: (s: string) => void;\n  /** @internal */\n  shutdownCallbacks: (() => Promise<void>)[] = [];\n  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];\n  #participantTasks: {\n    [id: string]: {\n      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;\n      result: Promise<void>;\n    };\n  } = {};\n  #logger: Logger;\n\n  constructor(\n    proc: JobProcess,\n    info: RunningJobInfo,\n    room: Room,\n    onConnect: () => void,\n    onShutdown: (s: string) => void,\n  ) {\n    this.#proc = proc;\n    this.#info = info;\n    this.#room = room;\n    this.#onConnect = onConnect;\n    this.#onShutdown = onShutdown;\n    this.onParticipantConnected = this.onParticipantConnected.bind(this);\n    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);\n    this.#logger = log().child({ info: this.#info });\n  }\n\n  get proc(): JobProcess {\n    return this.#proc;\n  }\n\n  get job(): proto.Job {\n    return this.#info.job;\n  }\n\n  /** @returns The room the agent was called into */\n  get room(): Room {\n    return this.#room;\n  }\n\n  /** @returns The agent's participant if connected to the room, otherwise `undefined` */\n  get agent(): LocalParticipant | undefined {\n    return this.#room.localParticipant;\n  }\n\n  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */\n  addShutdownCallback(callback: () => Promise<void>) {\n    this.shutdownCallbacks.push(callback);\n  }\n\n  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {\n    if (!this.#room.isConnected) {\n      throw new Error('room is not connected');\n    }\n\n    for (const p of this.#room.remoteParticipants.values()) {\n      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {\n        return p;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const onParticipantConnected = (participant: RemoteParticipant) => {\n        if (\n          (!identity || participant.identity === identity) &&\n          participant.info.kind != ParticipantKind.AGENT\n        ) {\n          clearHandlers();\n          resolve(participant);\n        }\n      };\n      const onDisconnected = () => {\n        clearHandlers();\n        reject(new Error('Room disconnected while waiting for participant'));\n      };\n\n      const clearHandlers = () => {\n        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);\n        this.#room.off(RoomEvent.Disconnected, onDisconnected);\n      };\n\n      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);\n      this.#room.on(RoomEvent.Disconnected, onDisconnected);\n    });\n  }\n\n  /**\n   * Connects the agent to the room.\n   *\n   * @remarks\n   * It is recommended to run this command as early in the function as possible, as executing it\n   * later may cause noticeable delay between user and agent joins.\n   *\n   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |\n   * @livekit/rtc-node} for more information about the parameters.\n   */\n  async connect(\n    e2ee?: E2EEOptions,\n    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,\n    rtcConfig?: RtcConfiguration,\n  ) {\n    const opts = {\n      e2ee,\n      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,\n      rtcConfig,\n      dynacast: false,\n    };\n\n    await this.#room.connect(this.#info.url, this.#info.token, opts);\n    this.#onConnect();\n\n    this.#room.remoteParticipants.forEach(this.onParticipantConnected);\n\n    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {\n      this.#room.remoteParticipants.forEach((p) => {\n        p.trackPublications.forEach((pub) => {\n          if (\n            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||\n            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)\n          ) {\n            pub.setSubscribed(true);\n          }\n        });\n      });\n    }\n  }\n\n  /**\n   * Gracefully shuts down the job, and runs all shutdown promises.\n   *\n   * @param reason - Optional reason for shutdown\n   */\n  shutdown(reason = '') {\n    this.#onShutdown(reason);\n  }\n\n  /** @internal */\n  onParticipantConnected(p: RemoteParticipant) {\n    for (const callback of this.#participantEntrypoints) {\n      if (this.#participantTasks[p.identity]?.callback == callback) {\n        this.#logger.warn(\n          'a participant has joined before a prior prticipant task matching the same identity has finished:',\n          p.identity,\n        );\n      }\n      const result = callback(this, p);\n      result.finally(() => delete this.#participantTasks[p.identity]);\n      this.#participantTasks[p.identity] = { callback, result };\n    }\n  }\n\n  /**\n   * Adds a promise to be awaited whenever a new participant joins the room.\n   *\n   * @throws {@link FunctionExistsError} if an entrypoint already exists\n   */\n  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {\n    if (this.#participantEntrypoints.includes(callback)) {\n      throw new FunctionExistsError('entrypoints cannot be added more than once');\n    }\n\n    this.#participantEntrypoints.push(callback);\n  }\n}\n\nexport class JobProcess {\n  #pid = process.pid;\n  userData: { [id: string]: unknown } = {};\n\n  get pid(): number {\n    return this.#pid;\n  }\n}\n\n/**\n * A request sent by the server to spawn a new agent job.\n *\n * @remarks\n * For most applications, this is best left to the default, which simply accepts the job and\n * handles the logic inside the entrypoint function. This class is useful for vetting which\n * requests should fill idle processes and which should be outright rejected.\n */\nexport class JobRequest {\n  #job: proto.Job;\n  #onReject: () => Promise<void>;\n  #onAccept: (args: JobAcceptArguments) => Promise<void>;\n\n  /** @internal */\n  constructor(\n    job: proto.Job,\n    onReject: () => Promise<void>,\n    onAccept: (args: JobAcceptArguments) => Promise<void>,\n  ) {\n    this.#job = job;\n    this.#onReject = onReject;\n    this.#onAccept = onAccept;\n  }\n\n  /** @returns The ID of the job, set by the LiveKit server */\n  get id(): string {\n    return this.#job.id;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get job(): proto.Job {\n    return this.#job;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get room(): proto.Room | undefined {\n    return this.#job.room;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get publisher(): proto.ParticipantInfo | undefined {\n    return this.#job.participant;\n  }\n\n  /** @returns The agent's name, as set in {@link WorkerOptions} */\n  get agentName(): string {\n    return this.#job.agentName;\n  }\n\n  /** Rejects the job. */\n  async reject() {\n    await this.#onReject();\n  }\n\n  /** Accepts the job, launching it on an idle child process. */\n  async accept(name = '', identity = '', metadata = '', attributes?: { [key: string]: string }) {\n    if (identity === '') identity = 'agent-' + this.id;\n\n    this.#onAccept({ name, identity, metadata, attributes });\n  }\n}\n"],"mappings":"AAWA,SAAS,iBAAiB,WAAW,iBAAiB;AAEtD,SAAS,WAAW;AAGb,IAAK,gBAAL,kBAAKA,mBAAL;AACL,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AAJU,SAAAA;AAAA,GAAA;AAsBL,MAAM,4BAA4B,MAAM;AAAA,EAC7C,YAAY,KAAc;AACxB,UAAM,GAAG;AACT,WAAO,eAAe,MAAM,WAAW,SAAS;AAAA,EAClD;AACF;AAGO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA,oBAA6C,CAAC;AAAA,EAC9C,0BAAwF,CAAC;AAAA,EACzF,oBAKI,CAAC;AAAA,EACL;AAAA,EAEA,YACE,MACA,MACA,MACA,WACA,YACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,SAAK,yBAAyB,KAAK,uBAAuB,KAAK,IAAI;AACnE,SAAK,MAAM,GAAG,UAAU,sBAAsB,KAAK,sBAAsB;AACzE,SAAK,UAAU,IAAI,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,QAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,oBAAoB,UAA+B;AACjD,SAAK,kBAAkB,KAAK,QAAQ;AAAA,EACtC;AAAA,EAEA,MAAM,mBAAmB,UAA+C;AACtE,QAAI,CAAC,KAAK,MAAM,aAAa;AAC3B,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AAEA,eAAW,KAAK,KAAK,MAAM,mBAAmB,OAAO,GAAG;AACtD,WAAK,CAAC,YAAY,EAAE,aAAa,aAAa,EAAE,KAAK,QAAQ,gBAAgB,OAAO;AAClF,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,yBAAyB,CAAC,gBAAmC;AACjE,aACG,CAAC,YAAY,YAAY,aAAa,aACvC,YAAY,KAAK,QAAQ,gBAAgB,OACzC;AACA,wBAAc;AACd,kBAAQ,WAAW;AAAA,QACrB;AAAA,MACF;AACA,YAAM,iBAAiB,MAAM;AAC3B,sBAAc;AACd,eAAO,IAAI,MAAM,iDAAiD,CAAC;AAAA,MACrE;AAEA,YAAM,gBAAgB,MAAM;AAC1B,aAAK,MAAM,IAAI,UAAU,sBAAsB,sBAAsB;AACrE,aAAK,MAAM,IAAI,UAAU,cAAc,cAAc;AAAA,MACvD;AAEA,WAAK,MAAM,GAAG,UAAU,sBAAsB,sBAAsB;AACpE,WAAK,MAAM,GAAG,UAAU,cAAc,cAAc;AAAA,IACtD,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,MACA,gBAA+B,uBAC/B,WACA;AACA,UAAM,OAAO;AAAA,MACX;AAAA,MACA,eAAe,iBAAiB;AAAA,MAChC;AAAA,MACA,UAAU;AAAA,IACZ;AAEA,UAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,MAAM,OAAO,IAAI;AAC/D,SAAK,WAAW;AAEhB,SAAK,MAAM,mBAAmB,QAAQ,KAAK,sBAAsB;AAEjE,QAAI,CAAC,oBAA0B,kBAAwB,EAAE,SAAS,aAAa,GAAG;AAChF,WAAK,MAAM,mBAAmB,QAAQ,CAAC,MAAM;AAC3C,UAAE,kBAAkB,QAAQ,CAAC,QAAQ;AACnC,cACG,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,cACrE,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,YACtE;AACA,gBAAI,cAAc,IAAI;AAAA,UACxB;AAAA,QACF,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,IAAI;AACpB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA;AAAA,EAGA,uBAAuB,GAAsB;AA9L/C;AA+LI,eAAW,YAAY,KAAK,yBAAyB;AACnD,YAAI,UAAK,kBAAkB,EAAE,QAAQ,MAAjC,mBAAoC,aAAY,UAAU;AAC5D,aAAK,QAAQ;AAAA,UACX;AAAA,UACA,EAAE;AAAA,QACJ;AAAA,MACF;AACA,YAAM,SAAS,SAAS,MAAM,CAAC;AAC/B,aAAO,QAAQ,MAAM,OAAO,KAAK,kBAAkB,EAAE,QAAQ,CAAC;AAC9D,WAAK,kBAAkB,EAAE,QAAQ,IAAI,EAAE,UAAU,OAAO;AAAA,IAC1D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,yBAAyB,UAAoE;AAC3F,QAAI,KAAK,wBAAwB,SAAS,QAAQ,GAAG;AACnD,YAAM,IAAI,oBAAoB,4CAA4C;AAAA,IAC5E;AAEA,SAAK,wBAAwB,KAAK,QAAQ;AAAA,EAC5C;AACF;AAEO,MAAM,WAAW;AAAA,EACtB,OAAO,QAAQ;AAAA,EACf,WAAsC,CAAC;AAAA,EAEvC,IAAI,MAAc;AAChB,WAAO,KAAK;AAAA,EACd;AACF;AAUO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,YACE,KACA,UACA,UACA;AACA,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,KAAa;AACf,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,MAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,OAA+B;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAA+C;AACjD,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB;AACtB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,MAAM,SAAS;AACb,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAO,IAAI,WAAW,IAAI,WAAW,IAAI,YAAwC;AAC5F,QAAI,aAAa,GAAI,YAAW,WAAW,KAAK;AAEhD,SAAK,UAAU,EAAE,MAAM,UAAU,UAAU,WAAW,CAAC;AAAA,EACzD;AACF;","names":["AutoSubscribe"]}
+{"version":3,"sources":["../src/job.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type * as proto from '@livekit/protocol';\nimport type {\n  E2EEOptions,\n  LocalParticipant,\n  RemoteParticipant,\n  Room,\n  RtcConfiguration,\n} from '@livekit/rtc-node';\nimport { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';\nimport type { Logger } from 'pino';\nimport { log } from './log.js';\n\n/** Which tracks, if any, should the agent automatically subscribe to? */\nexport enum AutoSubscribe {\n  SUBSCRIBE_ALL,\n  SUBSCRIBE_NONE,\n  VIDEO_ONLY,\n  AUDIO_ONLY,\n}\n\nexport type JobAcceptArguments = {\n  name: string;\n  identity: string;\n  metadata: string;\n  attributes?: { [key: string]: string };\n};\n\nexport type RunningJobInfo = {\n  acceptArguments: JobAcceptArguments;\n  job: proto.Job;\n  url: string;\n  token: string;\n};\n\n/** Attempted to add a function callback, but the function already exists. */\nexport class FunctionExistsError extends Error {\n  constructor(msg?: string) {\n    super(msg);\n    Object.setPrototypeOf(this, new.target.prototype);\n  }\n}\n\n/** The job and environment context as seen by the agent, accessible by the entrypoint function. */\nexport class JobContext {\n  #proc: JobProcess;\n  #info: RunningJobInfo;\n  #room: Room;\n  #onConnect: () => void;\n  #onShutdown: (s: string) => void;\n  /** @internal */\n  shutdownCallbacks: (() => Promise<void>)[] = [];\n  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];\n  #participantTasks: {\n    [id: string]: {\n      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;\n      result: Promise<void>;\n    };\n  } = {};\n  #logger: Logger;\n\n  constructor(\n    proc: JobProcess,\n    info: RunningJobInfo,\n    room: Room,\n    onConnect: () => void,\n    onShutdown: (s: string) => void,\n  ) {\n    this.#proc = proc;\n    this.#info = info;\n    this.#room = room;\n    this.#onConnect = onConnect;\n    this.#onShutdown = onShutdown;\n    this.onParticipantConnected = this.onParticipantConnected.bind(this);\n    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);\n    this.#logger = log().child({ info: this.#info });\n  }\n\n  get proc(): JobProcess {\n    return this.#proc;\n  }\n\n  get job(): proto.Job {\n    return this.#info.job;\n  }\n\n  /** @returns The room the agent was called into */\n  get room(): Room {\n    return this.#room;\n  }\n\n  /** @returns The agent's participant if connected to the room, otherwise `undefined` */\n  get agent(): LocalParticipant | undefined {\n    return this.#room.localParticipant;\n  }\n\n  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */\n  addShutdownCallback(callback: () => Promise<void>) {\n    this.shutdownCallbacks.push(callback);\n  }\n\n  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {\n    if (!this.#room.isConnected) {\n      throw new Error('room is not connected');\n    }\n\n    for (const p of this.#room.remoteParticipants.values()) {\n      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {\n        return p;\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const onParticipantConnected = (participant: RemoteParticipant) => {\n        if (\n          (!identity || participant.identity === identity) &&\n          participant.info.kind != ParticipantKind.AGENT\n        ) {\n          clearHandlers();\n          resolve(participant);\n        }\n      };\n      const onDisconnected = () => {\n        clearHandlers();\n        reject(new Error('Room disconnected while waiting for participant'));\n      };\n\n      const clearHandlers = () => {\n        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);\n        this.#room.off(RoomEvent.Disconnected, onDisconnected);\n      };\n\n      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);\n      this.#room.on(RoomEvent.Disconnected, onDisconnected);\n    });\n  }\n\n  /**\n   * Connects the agent to the room.\n   *\n   * @remarks\n   * It is recommended to run this command as early in the function as possible, as executing it\n   * later may cause noticeable delay between user and agent joins.\n   *\n   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |\n   * @livekit/rtc-node} for more information about the parameters.\n   */\n  async connect(\n    e2ee?: E2EEOptions,\n    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,\n    rtcConfig?: RtcConfiguration,\n  ) {\n    const opts = {\n      e2ee,\n      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,\n      rtcConfig,\n      dynacast: false,\n    };\n\n    await this.#room.connect(this.#info.url, this.#info.token, opts);\n    this.#onConnect();\n\n    this.#room.remoteParticipants.forEach(this.onParticipantConnected);\n\n    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {\n      this.#room.remoteParticipants.forEach((p) => {\n        p.trackPublications.forEach((pub) => {\n          if (\n            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||\n            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)\n          ) {\n            pub.setSubscribed(true);\n          }\n        });\n      });\n    }\n  }\n\n  /**\n   * Gracefully shuts down the job, and runs all shutdown promises.\n   *\n   * @param reason - Optional reason for shutdown\n   */\n  shutdown(reason = '') {\n    this.#onShutdown(reason);\n  }\n\n  /** @internal */\n  onParticipantConnected(p: RemoteParticipant) {\n    for (const callback of this.#participantEntrypoints) {\n      if (this.#participantTasks[p.identity!]?.callback == callback) {\n        this.#logger.warn(\n          'a participant has joined before a prior prticipant task matching the same identity has finished:',\n          p.identity,\n        );\n      }\n      const result = callback(this, p);\n      result.finally(() => delete this.#participantTasks[p.identity!]);\n      this.#participantTasks[p.identity!] = { callback, result };\n    }\n  }\n\n  /**\n   * Adds a promise to be awaited whenever a new participant joins the room.\n   *\n   * @throws {@link FunctionExistsError} if an entrypoint already exists\n   */\n  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {\n    if (this.#participantEntrypoints.includes(callback)) {\n      throw new FunctionExistsError('entrypoints cannot be added more than once');\n    }\n\n    this.#participantEntrypoints.push(callback);\n  }\n}\n\nexport class JobProcess {\n  #pid = process.pid;\n  userData: { [id: string]: unknown } = {};\n\n  get pid(): number {\n    return this.#pid;\n  }\n}\n\n/**\n * A request sent by the server to spawn a new agent job.\n *\n * @remarks\n * For most applications, this is best left to the default, which simply accepts the job and\n * handles the logic inside the entrypoint function. This class is useful for vetting which\n * requests should fill idle processes and which should be outright rejected.\n */\nexport class JobRequest {\n  #job: proto.Job;\n  #onReject: () => Promise<void>;\n  #onAccept: (args: JobAcceptArguments) => Promise<void>;\n\n  /** @internal */\n  constructor(\n    job: proto.Job,\n    onReject: () => Promise<void>,\n    onAccept: (args: JobAcceptArguments) => Promise<void>,\n  ) {\n    this.#job = job;\n    this.#onReject = onReject;\n    this.#onAccept = onAccept;\n  }\n\n  /** @returns The ID of the job, set by the LiveKit server */\n  get id(): string {\n    return this.#job.id;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get job(): proto.Job {\n    return this.#job;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get room(): proto.Room | undefined {\n    return this.#job.room;\n  }\n\n  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */\n  get publisher(): proto.ParticipantInfo | undefined {\n    return this.#job.participant;\n  }\n\n  /** @returns The agent's name, as set in {@link WorkerOptions} */\n  get agentName(): string {\n    return this.#job.agentName;\n  }\n\n  /** Rejects the job. */\n  async reject() {\n    await this.#onReject();\n  }\n\n  /** Accepts the job, launching it on an idle child process. */\n  async accept(name = '', identity = '', metadata = '', attributes?: { [key: string]: string }) {\n    if (identity === '') identity = 'agent-' + this.id;\n\n    this.#onAccept({ name, identity, metadata, attributes });\n  }\n}\n"],"mappings":"AAWA,SAAS,iBAAiB,WAAW,iBAAiB;AAEtD,SAAS,WAAW;AAGb,IAAK,gBAAL,kBAAKA,mBAAL;AACL,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AACA,EAAAA,8BAAA;AAJU,SAAAA;AAAA,GAAA;AAsBL,MAAM,4BAA4B,MAAM;AAAA,EAC7C,YAAY,KAAc;AACxB,UAAM,GAAG;AACT,WAAO,eAAe,MAAM,WAAW,SAAS;AAAA,EAClD;AACF;AAGO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA,oBAA6C,CAAC;AAAA,EAC9C,0BAAwF,CAAC;AAAA,EACzF,oBAKI,CAAC;AAAA,EACL;AAAA,EAEA,YACE,MACA,MACA,MACA,WACA,YACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,SAAK,yBAAyB,KAAK,uBAAuB,KAAK,IAAI;AACnE,SAAK,MAAM,GAAG,UAAU,sBAAsB,KAAK,sBAAsB;AACzE,SAAK,UAAU,IAAI,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAAA,EACjD;AAAA,EAEA,IAAI,OAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,QAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,oBAAoB,UAA+B;AACjD,SAAK,kBAAkB,KAAK,QAAQ;AAAA,EACtC;AAAA,EAEA,MAAM,mBAAmB,UAA+C;AACtE,QAAI,CAAC,KAAK,MAAM,aAAa;AAC3B,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AAEA,eAAW,KAAK,KAAK,MAAM,mBAAmB,OAAO,GAAG;AACtD,WAAK,CAAC,YAAY,EAAE,aAAa,aAAa,EAAE,KAAK,QAAQ,gBAAgB,OAAO;AAClF,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,yBAAyB,CAAC,gBAAmC;AACjE,aACG,CAAC,YAAY,YAAY,aAAa,aACvC,YAAY,KAAK,QAAQ,gBAAgB,OACzC;AACA,wBAAc;AACd,kBAAQ,WAAW;AAAA,QACrB;AAAA,MACF;AACA,YAAM,iBAAiB,MAAM;AAC3B,sBAAc;AACd,eAAO,IAAI,MAAM,iDAAiD,CAAC;AAAA,MACrE;AAEA,YAAM,gBAAgB,MAAM;AAC1B,aAAK,MAAM,IAAI,UAAU,sBAAsB,sBAAsB;AACrE,aAAK,MAAM,IAAI,UAAU,cAAc,cAAc;AAAA,MACvD;AAEA,WAAK,MAAM,GAAG,UAAU,sBAAsB,sBAAsB;AACpE,WAAK,MAAM,GAAG,UAAU,cAAc,cAAc;AAAA,IACtD,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,MACA,gBAA+B,uBAC/B,WACA;AACA,UAAM,OAAO;AAAA,MACX;AAAA,MACA,eAAe,iBAAiB;AAAA,MAChC;AAAA,MACA,UAAU;AAAA,IACZ;AAEA,UAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,MAAM,OAAO,IAAI;AAC/D,SAAK,WAAW;AAEhB,SAAK,MAAM,mBAAmB,QAAQ,KAAK,sBAAsB;AAEjE,QAAI,CAAC,oBAA0B,kBAAwB,EAAE,SAAS,aAAa,GAAG;AAChF,WAAK,MAAM,mBAAmB,QAAQ,CAAC,MAAM;AAC3C,UAAE,kBAAkB,QAAQ,CAAC,QAAQ;AACnC,cACG,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,cACrE,kBAAkB,sBAA4B,IAAI,SAAS,UAAU,YACtE;AACA,gBAAI,cAAc,IAAI;AAAA,UACxB;AAAA,QACF,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,IAAI;AACpB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA;AAAA,EAGA,uBAAuB,GAAsB;AA9L/C;AA+LI,eAAW,YAAY,KAAK,yBAAyB;AACnD,YAAI,UAAK,kBAAkB,EAAE,QAAS,MAAlC,mBAAqC,aAAY,UAAU;AAC7D,aAAK,QAAQ;AAAA,UACX;AAAA,UACA,EAAE;AAAA,QACJ;AAAA,MACF;AACA,YAAM,SAAS,SAAS,MAAM,CAAC;AAC/B,aAAO,QAAQ,MAAM,OAAO,KAAK,kBAAkB,EAAE,QAAS,CAAC;AAC/D,WAAK,kBAAkB,EAAE,QAAS,IAAI,EAAE,UAAU,OAAO;AAAA,IAC3D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,yBAAyB,UAAoE;AAC3F,QAAI,KAAK,wBAAwB,SAAS,QAAQ,GAAG;AACnD,YAAM,IAAI,oBAAoB,4CAA4C;AAAA,IAC5E;AAEA,SAAK,wBAAwB,KAAK,QAAQ;AAAA,EAC5C;AACF;AAEO,MAAM,WAAW;AAAA,EACtB,OAAO,QAAQ;AAAA,EACf,WAAsC,CAAC;AAAA,EAEvC,IAAI,MAAc;AAChB,WAAO,KAAK;AAAA,EACd;AACF;AAUO,MAAM,WAAW;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,YACE,KACA,UACA,UACA;AACA,SAAK,OAAO;AACZ,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,KAAa;AACf,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,MAAiB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,OAA+B;AACjC,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAA+C;AACjD,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,IAAI,YAAoB;AACtB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA;AAAA,EAGA,MAAM,SAAS;AACb,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAO,IAAI,WAAW,IAAI,WAAW,IAAI,YAAwC;AAC5F,QAAI,aAAa,GAAI,YAAW,WAAW,KAAK;AAEhD,SAAK,UAAU,EAAE,MAAM,UAAU,UAAU,WAAW,CAAC;AAAA,EACzD;AACF;","names":["AutoSubscribe"]}

package/dist/multimodal/multimodal_agent.cjs

@@ -53,15 +53,19 @@ class MultimodalAgent extends import_node_events.EventEmitter {
   linkedParticipant = null;
   subscribedTrack = null;
   readMicroTask = null;
+  #textResponseRetries = 0;
+  #maxTextResponseRetries;
   constructor({
     model,
     chatCtx,
-    fncCtx
+    fncCtx,
+    maxTextResponseRetries = 5
   }) {
     super();
     this.model = model;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.#maxTextResponseRetries = maxTextResponseRetries;
   }
   #participant = null;
   #agentPublication = null;
@@ -190,6 +194,7 @@ class MultimodalAgent extends import_node_events.EventEmitter {
       this.#started = true;
       this.#session.on("response_content_added", (message) => {
         var _a2;
+        if (message.contentType === "text") return;
         const trFwd = new import_transcription.BasicTranscriptionForwarder(
           this.room,
           this.room.localParticipant.identity,
@@ -205,6 +210,26 @@ class MultimodalAgent extends import_node_events.EventEmitter {
         );
         this.#playingHandle = handle;
       });
+      this.#session.on("response_content_done", (message) => {
+        if (message.contentType === "text") {
+          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {
+            throw new Error(
+              `The OpenAI Realtime API returned a text response after ${this.#maxTextResponseRetries} retries. Please try to reduce the number of text system or assistant messages in the chat context.`
+            );
+          }
+          this.#textResponseRetries++;
+          this.#logger.child({
+            itemId: message.itemId,
+            text: message.text,
+            retries: this.#textResponseRetries
+          }).warn(
+            "The OpenAI Realtime API returned a text response instead of audio. Attempting to recover to audio mode..."
+          );
+          this.#session.recoverFromTextResponse(message.itemId);
+        } else {
+          this.#textResponseRetries = 0;
+        }
+      });
       this.#session.on("input_speech_committed", (ev) => {
         var _a2, _b;
         const participantIdentity = (_a2 = this.linkedParticipant) == null ? void 0 : _a2.identity;
@@ -334,9 +359,8 @@ class MultimodalAgent extends import_node_events.EventEmitter {
     });
   }
   #getLocalTrackSid() {
-    var _a;
     if (!this.#localTrackSid && this.room && this.room.localParticipant) {
-      this.#localTrackSid = (0, import_utils.findMicroTrackId)(this.room, (_a = this.room.localParticipant) == null ? void 0 : _a.identity);
+      this.#localTrackSid = (0, import_utils.findMicroTrackId)(this.room, this.room.localParticipant.identity);
     }
     return this.#localTrackSid;
   }

package/dist/multimodal/multimodal_agent.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { MultimodalLLMMetrics } from '../metrics/base.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: Promise<void> | null = null;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        this.emit('user_started_speaking');\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {\n        this.emit('metrics_collected', metrics);\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    this.readMicroTask = new Promise<void>((resolve, reject) => {\n      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n        .then(resolve)\n        .catch(reject);\n    });\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAOO;AACP,yBAA6B;AAC7B,mBAAgC;AAChC,UAAqB;AACrB,iBAAoB;AAEpB,2BAA4C;AAC5C,mBAAiC;AACjC,2BAAiD;AAM1C,MAAe,wBAAwB,gCAAa;AAM3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,gCAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAsC;AAAA,EAEtC,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,cAAU,gBAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAzIlD;AA0IM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,0BAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAQ;AAAA,MAC5C,CAAC;AACD,WAAK;AAAA,QACH,0BAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,4BAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,0BAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,4BAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oCAAoB;AACxC,cAAQ,SAAS,4BAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAQ;AAAA,QAC5C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAQ;AAC1C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AA5OnE,YAAAC;AA8OQ,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AAhQ9D,YAAAA,KAAA;AAkQQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AA5Q5E,YAAAA,KAAA;AA8QQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AA9R5D,YAAAA,KAAA;AA+RQ,aAAK,KAAK,uBAAuB;AACjC,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,WAAK,SAAS,GAAG,qBAAqB,CAAC,YAAkC;AACvE,aAAK,KAAK,qBAAqB,OAAO;AAAA,MACxC,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AArYJ;AAsYI,QACE,YAAY,WAAW,4BAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,SAAK,gBAAgB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC1D,0BAAoB,IAAI,4BAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,oBAAmC;AAnarC;AAoaI,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,qBAAiB,+BAAiB,KAAK,OAAM,UAAK,KAAK,qBAAV,mBAA4B,QAAQ;AAAA,IACxF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AAhbV;AAibI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AAtd/B;AAudI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAW,qBAAqB;AAChF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}
+{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { MultimodalLLMMetrics } from '../metrics/base.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n  abstract recoverFromTextResponse(itemId: string): void;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: Promise<void> | null = null;\n\n  #textResponseRetries = 0;\n  #maxTextResponseRetries: number;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n    maxTextResponseRetries = 5,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n    maxTextResponseRetries?: number;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.#maxTextResponseRetries = maxTextResponseRetries;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity!);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity!);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity!);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        if (message.contentType === 'text') return;\n\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity!,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_done', (message: any) => {\n        // openai.realtime.RealtimeContent\n        if (message.contentType === 'text') {\n          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {\n            throw new Error(\n              'The OpenAI Realtime API returned a text response ' +\n                `after ${this.#maxTextResponseRetries} retries. ` +\n                'Please try to reduce the number of text system or ' +\n                'assistant messages in the chat context.',\n            );\n          }\n\n          this.#textResponseRetries++;\n          this.#logger\n            .child({\n              itemId: message.itemId,\n              text: message.text,\n              retries: this.#textResponseRetries,\n            })\n            .warn(\n              'The OpenAI Realtime API returned a text response instead of audio. ' +\n                'Attempting to recover to audio mode...',\n            );\n          this.#session!.recoverFromTextResponse(message.itemId);\n        } else {\n          this.#textResponseRetries = 0;\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        this.emit('user_started_speaking');\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {\n        this.emit('metrics_collected', metrics);\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    this.readMicroTask = new Promise<void>((resolve, reject) => {\n      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n        .then(resolve)\n        .catch(reject);\n    });\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant!.identity!);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes![AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,sBAOO;AACP,yBAA6B;AAC7B,mBAAgC;AAChC,UAAqB;AACrB,iBAAoB;AAEpB,2BAA4C;AAC5C,mBAAiC;AACjC,2BAAiD;AAM1C,MAAe,wBAAwB,gCAAa;AAO3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,gCAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAsC;AAAA,EAEtC,uBAAuB;AAAA,EACvB;AAAA,EAEA,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA,yBAAyB;AAAA,EAC3B,GAKG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,cAAU,gBAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAhJlD;AAiJM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,0BAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAS;AAAA,MAC7C,CAAC;AACD,WAAK;AAAA,QACH,0BAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,4BAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,0BAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,4BAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oCAAoB;AACxC,cAAQ,SAAS,4BAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAS;AAAA,QAC7C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAS;AAC3C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AAnPnE,YAAAC;AAqPQ,YAAI,QAAQ,gBAAgB,OAAQ;AAEpC,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,YAAiB;AAE1D,YAAI,QAAQ,gBAAgB,QAAQ;AAClC,cAAI,KAAK,wBAAwB,KAAK,yBAAyB;AAC7D,kBAAM,IAAI;AAAA,cACR,0DACW,KAAK,uBAAuB;AAAA,YAGzC;AAAA,UACF;AAEA,eAAK;AACL,eAAK,QACF,MAAM;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,MAAM,QAAQ;AAAA,YACd,SAAS,KAAK;AAAA,UAChB,CAAC,EACA;AAAA,YACC;AAAA,UAEF;AACF,eAAK,SAAU,wBAAwB,QAAQ,MAAM;AAAA,QACvD,OAAO;AACL,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AAvS9D,YAAAA,KAAA;AAySQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AAnT5E,YAAAA,KAAA;AAqTQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AArU5D,YAAAA,KAAA;AAsUQ,aAAK,KAAK,uBAAuB;AACjC,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,WAAK,SAAS,GAAG,qBAAqB,CAAC,YAAkC;AACvE,aAAK,KAAK,qBAAqB,OAAO;AAAA,MACxC,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AA5aJ;AA6aI,QACE,YAAY,WAAW,4BAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,SAAK,gBAAgB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC1D,0BAAoB,IAAI,4BAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,oBAAmC;AACjC,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,qBAAiB,+BAAiB,KAAK,MAAM,KAAK,KAAK,iBAAkB,QAAS;AAAA,IACzF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AAvdV;AAwdI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AA7f/B;AA8fI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAY,qBAAqB;AACjF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}

package/dist/multimodal/multimodal_agent.d.ts

@@ -10,6 +10,7 @@ export declare abstract class RealtimeSession extends EventEmitter {
     abstract conversation: any;
     abstract inputAudioBuffer: any;
     abstract fncCtx: llm.FunctionContext | undefined;
+    abstract recoverFromTextResponse(itemId: string): void;
 }
 /**
  * @internal
@@ -33,10 +34,11 @@ export declare class MultimodalAgent extends EventEmitter {
     linkedParticipant: RemoteParticipant | null;
     subscribedTrack: RemoteAudioTrack | null;
     readMicroTask: Promise<void> | null;
-    constructor({ model, chatCtx, fncCtx, }: {
+    constructor({ model, chatCtx, fncCtx, maxTextResponseRetries, }: {
         model: RealtimeModel;
         chatCtx?: llm.ChatContext;
         fncCtx?: llm.FunctionContext;
+        maxTextResponseRetries?: number;
     });
     get fncCtx(): llm.FunctionContext | undefined;
     set fncCtx(ctx: llm.FunctionContext | undefined);

package/dist/multimodal/multimodal_agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"multimodal_agent.d.ts","sourceRoot":"","sources":["../../src/multimodal/multimodal_agent.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAEV,gBAAgB,EAChB,iBAAiB,EAGjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,GAAG,MAAM,iBAAiB,CAAC;AAOvC;;;GAGG;AACH,8BAAsB,eAAgB,SAAQ,YAAY;IAExD,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC;IAE3B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;CAClD;AAED;;;GAGG;AACH,8BAAsB,aAAa;IAEjC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,eAAe;IAC/C,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD,YAAY;AACZ,qBAAa,eAAgB,SAAQ,YAAY;;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,IAAI,GAAG,IAAI,CAAQ;IACzB,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAQ;IACnD,eAAe,EAAE,gBAAgB,GAAG,IAAI,CAAQ;IAChD,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;gBAE/B,EACV,KAAK,EACL,OAAO,EACP,MAAM,GACP,EAAE;QACD,KAAK,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;KAC9B;IAsBD,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAK9C;IA6BD,KAAK,CACH,IAAI,EAAE,IAAI,EACV,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW,GACpD,OAAO,CAAC,eAAe,CAAC;CAyV5B"}
+{"version":3,"file":"multimodal_agent.d.ts","sourceRoot":"","sources":["../../src/multimodal/multimodal_agent.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAEV,gBAAgB,EAChB,iBAAiB,EAGjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,GAAG,MAAM,iBAAiB,CAAC;AAOvC;;;GAGG;AACH,8BAAsB,eAAgB,SAAQ,YAAY;IAExD,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC;IAE3B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;CACvD;AAED;;;GAGG;AACH,8BAAsB,aAAa;IAEjC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,eAAe;IAC/C,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD,YAAY;AACZ,qBAAa,eAAgB,SAAQ,YAAY;;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,IAAI,GAAG,IAAI,CAAQ;IACzB,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAQ;IACnD,eAAe,EAAE,gBAAgB,GAAG,IAAI,CAAQ;IAChD,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;gBAK/B,EACV,KAAK,EACL,OAAO,EACP,MAAM,EACN,sBAA0B,GAC3B,EAAE;QACD,KAAK,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;QAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;KACjC;IAuBD,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAK9C;IA6BD,KAAK,CACH,IAAI,EAAE,IAAI,EACV,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW,GACpD,OAAO,CAAC,eAAe,CAAC;CAyX5B"}

package/dist/multimodal/multimodal_agent.js

@@ -24,15 +24,19 @@ class MultimodalAgent extends EventEmitter {
   linkedParticipant = null;
   subscribedTrack = null;
   readMicroTask = null;
+  #textResponseRetries = 0;
+  #maxTextResponseRetries;
   constructor({
     model,
     chatCtx,
-    fncCtx
+    fncCtx,
+    maxTextResponseRetries = 5
   }) {
     super();
     this.model = model;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.#maxTextResponseRetries = maxTextResponseRetries;
   }
   #participant = null;
   #agentPublication = null;
@@ -161,6 +165,7 @@ class MultimodalAgent extends EventEmitter {
       this.#started = true;
       this.#session.on("response_content_added", (message) => {
         var _a2;
+        if (message.contentType === "text") return;
         const trFwd = new BasicTranscriptionForwarder(
           this.room,
           this.room.localParticipant.identity,
@@ -176,6 +181,26 @@ class MultimodalAgent extends EventEmitter {
         );
         this.#playingHandle = handle;
       });
+      this.#session.on("response_content_done", (message) => {
+        if (message.contentType === "text") {
+          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {
+            throw new Error(
+              `The OpenAI Realtime API returned a text response after ${this.#maxTextResponseRetries} retries. Please try to reduce the number of text system or assistant messages in the chat context.`
+            );
+          }
+          this.#textResponseRetries++;
+          this.#logger.child({
+            itemId: message.itemId,
+            text: message.text,
+            retries: this.#textResponseRetries
+          }).warn(
+            "The OpenAI Realtime API returned a text response instead of audio. Attempting to recover to audio mode..."
+          );
+          this.#session.recoverFromTextResponse(message.itemId);
+        } else {
+          this.#textResponseRetries = 0;
+        }
+      });
       this.#session.on("input_speech_committed", (ev) => {
         var _a2, _b;
         const participantIdentity = (_a2 = this.linkedParticipant) == null ? void 0 : _a2.identity;
@@ -305,9 +330,8 @@ class MultimodalAgent extends EventEmitter {
     });
   }
   #getLocalTrackSid() {
-    var _a;
     if (!this.#localTrackSid && this.room && this.room.localParticipant) {
-      this.#localTrackSid = findMicroTrackId(this.room, (_a = this.room.localParticipant) == null ? void 0 : _a.identity);
+      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant.identity);
     }
     return this.#localTrackSid;
   }

package/dist/multimodal/multimodal_agent.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { MultimodalLLMMetrics } from '../metrics/base.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: Promise<void> | null = null;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        this.emit('user_started_speaking');\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {\n        this.emit('metrics_collected', metrics);\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    this.readMicroTask = new Promise<void>((resolve, reject) => {\n      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n        .then(resolve)\n        .catch(reject);\n    });\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":"AAWA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,oBAAoB;AAC7B,SAAS,uBAAuB;AAChC,YAAY,SAAS;AACrB,SAAS,WAAW;AAEpB,SAAS,mCAAmC;AAC5C,SAAS,wBAAwB;AACjC,SAAS,oBAAwC;AAM1C,MAAe,wBAAwB,aAAa;AAM3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,aAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAsC;AAAA,EAEtC,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,UAAU,IAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAzIlD;AA0IM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,UAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAQ;AAAA,MAC5C,CAAC;AACD,WAAK;AAAA,QACH,UAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,YAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,UAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,YAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gBAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oBAAoB;AACxC,cAAQ,SAAS,YAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAQ;AAAA,QAC5C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAQ;AAC1C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AA5OnE,YAAAC;AA8OQ,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AAhQ9D,YAAAA,KAAA;AAkQQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AA5Q5E,YAAAA,KAAA;AA8QQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AA9R5D,YAAAA,KAAA;AA+RQ,aAAK,KAAK,uBAAuB;AACjC,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,WAAK,SAAS,GAAG,qBAAqB,CAAC,YAAkC;AACvE,aAAK,KAAK,qBAAqB,OAAO;AAAA,MACxC,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,YAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,YAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AArYJ;AAsYI,QACE,YAAY,WAAW,YAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,SAAK,gBAAgB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC1D,0BAAoB,IAAI,YAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,oBAAmC;AAnarC;AAoaI,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,iBAAiB,iBAAiB,KAAK,OAAM,UAAK,KAAK,qBAAV,mBAA4B,QAAQ;AAAA,IACxF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AAhbV;AAibI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AAtd/B;AAudI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAW,qBAAqB;AAChF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}
+{"version":3,"sources":["../../src/multimodal/multimodal_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrack,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  AudioStream,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport { AudioByteStream } from '../audio.js';\nimport * as llm from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { MultimodalLLMMetrics } from '../metrics/base.js';\nimport { BasicTranscriptionForwarder } from '../transcription.js';\nimport { findMicroTrackId } from '../utils.js';\nimport { AgentPlayout, type PlayoutHandle } from './agent_playout.js';\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeSession extends EventEmitter {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract conversation: any; // openai.realtime.Conversation\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer\n  abstract fncCtx: llm.FunctionContext | undefined;\n  abstract recoverFromTextResponse(itemId: string): void;\n}\n\n/**\n * @internal\n * @beta\n */\nexport abstract class RealtimeModel {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions\n  abstract close(): Promise<void>;\n  abstract sampleRate: number;\n  abstract numChannels: number;\n  abstract inFrameSize: number;\n  abstract outFrameSize: number;\n}\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\n\n/** @beta */\nexport class MultimodalAgent extends EventEmitter {\n  model: RealtimeModel;\n  room: Room | null = null;\n  linkedParticipant: RemoteParticipant | null = null;\n  subscribedTrack: RemoteAudioTrack | null = null;\n  readMicroTask: Promise<void> | null = null;\n\n  #textResponseRetries = 0;\n  #maxTextResponseRetries: number;\n\n  constructor({\n    model,\n    chatCtx,\n    fncCtx,\n    maxTextResponseRetries = 5,\n  }: {\n    model: RealtimeModel;\n    chatCtx?: llm.ChatContext;\n    fncCtx?: llm.FunctionContext;\n    maxTextResponseRetries?: number;\n  }) {\n    super();\n    this.model = model;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.#maxTextResponseRetries = maxTextResponseRetries;\n  }\n\n  #participant: RemoteParticipant | string | null = null;\n  #agentPublication: LocalTrackPublication | null = null;\n  #localTrackSid: string | null = null;\n  #localSource: AudioSource | null = null;\n  #agentPlayout: AgentPlayout | null = null;\n  #playingHandle: PlayoutHandle | undefined = undefined;\n  #logger = log();\n  #session: RealtimeSession | null = null;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #chatCtx: llm.ChatContext | undefined = undefined;\n\n  #_started: boolean = false;\n  #_pendingFunctionCalls: Set<string> = new Set();\n  #_speaking: boolean = false;\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n    if (this.#session) {\n      this.#session.fncCtx = ctx;\n    }\n  }\n\n  get #pendingFunctionCalls(): Set<string> {\n    return this.#_pendingFunctionCalls;\n  }\n\n  set #pendingFunctionCalls(calls: Set<string>) {\n    this.#_pendingFunctionCalls = calls;\n    this.#updateState();\n  }\n\n  get #speaking(): boolean {\n    return this.#_speaking;\n  }\n\n  set #speaking(isSpeaking: boolean) {\n    this.#_speaking = isSpeaking;\n    this.#updateState();\n  }\n\n  get #started(): boolean {\n    return this.#_started;\n  }\n\n  set #started(started: boolean) {\n    this.#_started = started;\n    this.#updateState();\n  }\n\n  start(\n    room: Room,\n    participant: RemoteParticipant | string | null = null,\n  ): Promise<RealtimeSession> {\n    return new Promise(async (resolve, reject) => {\n      if (this.#started) {\n        reject(new Error('MultimodalAgent already started'));\n      }\n      this.#updateState();\n\n      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n        // automatically link to the first participant that connects, if not already linked\n        if (this.linkedParticipant) {\n          return;\n        }\n        this.#linkParticipant(participant.identity!);\n      });\n      room.on(\n        RoomEvent.TrackPublished,\n        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {\n          if (\n            this.linkedParticipant &&\n            participant.identity === this.linkedParticipant.identity &&\n            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&\n            !trackPublication.subscribed\n          ) {\n            trackPublication.setSubscribed(true);\n          }\n        },\n      );\n      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));\n\n      this.room = room;\n      this.#participant = participant;\n\n      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);\n      this.#agentPlayout = new AgentPlayout(\n        this.#localSource,\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n        this.model.outFrameSize,\n      );\n      const onPlayoutStarted = () => {\n        this.emit('agent_started_speaking');\n        this.#speaking = true;\n      };\n\n      const onPlayoutStopped = (interrupted: boolean) => {\n        this.emit('agent_stopped_speaking');\n        this.#speaking = false;\n        if (this.#playingHandle) {\n          let text = this.#playingHandle.transcriptionFwd.text;\n          if (interrupted) {\n            text += '…';\n          }\n          const msg = llm.ChatMessage.create({\n            role: llm.ChatRole.ASSISTANT,\n            text,\n          });\n\n          if (interrupted) {\n            this.emit('agent_speech_interrupted', msg);\n          } else {\n            this.emit('agent_speech_committed', msg);\n          }\n          this.#logger.child({ transcription: text, interrupted }).debug('committed agent speech');\n        }\n      };\n\n      this.#agentPlayout.on('playout_started', onPlayoutStarted);\n      this.#agentPlayout.on('playout_stopped', onPlayoutStopped);\n\n      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);\n      const options = new TrackPublishOptions();\n      options.source = TrackSource.SOURCE_MICROPHONE;\n      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;\n      if (!this.#agentPublication) {\n        this.#logger.error('Failed to publish track');\n        reject(new Error('Failed to publish track'));\n        return;\n      }\n\n      await this.#agentPublication.waitForSubscription();\n\n      if (participant) {\n        if (typeof participant === 'string') {\n          this.#linkParticipant(participant);\n        } else {\n          this.#linkParticipant(participant.identity!);\n        }\n      } else {\n        // No participant specified, try to find the first participant in the room\n        for (const participant of room.remoteParticipants.values()) {\n          this.#linkParticipant(participant.identity!);\n          break;\n        }\n      }\n\n      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });\n      this.#started = true;\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_added', (message: any) => {\n        // openai.realtime.RealtimeContent\n        if (message.contentType === 'text') return;\n\n        const trFwd = new BasicTranscriptionForwarder(\n          this.room!,\n          this.room!.localParticipant!.identity!,\n          this.#getLocalTrackSid()!,\n          message.responseId,\n        );\n\n        const handle = this.#agentPlayout?.play(\n          message.itemId,\n          message.contentIndex,\n          trFwd,\n          message.textStream,\n          message.audioStream,\n        );\n        this.#playingHandle = handle;\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('response_content_done', (message: any) => {\n        // openai.realtime.RealtimeContent\n        if (message.contentType === 'text') {\n          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {\n            throw new Error(\n              'The OpenAI Realtime API returned a text response ' +\n                `after ${this.#maxTextResponseRetries} retries. ` +\n                'Please try to reduce the number of text system or ' +\n                'assistant messages in the chat context.',\n            );\n          }\n\n          this.#textResponseRetries++;\n          this.#logger\n            .child({\n              itemId: message.itemId,\n              text: message.text,\n              retries: this.#textResponseRetries,\n            })\n            .warn(\n              'The OpenAI Realtime API returned a text response instead of audio. ' +\n                'Attempting to recover to audio mode...',\n            );\n          this.#session!.recoverFromTextResponse(message.itemId);\n        } else {\n          this.#textResponseRetries = 0;\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_committed', (ev: any) => {\n        // openai.realtime.InputSpeechCommittedEvent\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('input_speech_transcription_completed', (ev: any) => {\n        // openai.realtime.InputSpeechTranscriptionCompletedEvent\n        const transcription = ev.transcript;\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);\n        } else {\n          this.#logger.error('Participant or track not set');\n        }\n        const userMsg = llm.ChatMessage.create({\n          role: llm.ChatRole.USER,\n          text: transcription,\n        });\n        this.emit('user_speech_committed', userMsg);\n        this.#logger.child({ transcription }).debug('committed user speech');\n      });\n\n      this.#session.on('input_speech_started', (ev: any) => {\n        this.emit('user_started_speaking');\n        if (this.#playingHandle && !this.#playingHandle.done) {\n          this.#playingHandle.interrupt();\n\n          this.#session!.conversation.item.truncate(\n            this.#playingHandle.itemId,\n            this.#playingHandle.contentIndex,\n            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),\n          );\n\n          this.#playingHandle = undefined;\n        }\n\n        const participantIdentity = this.linkedParticipant?.identity;\n        const trackSid = this.subscribedTrack?.sid;\n        if (participantIdentity && trackSid) {\n          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);\n        }\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#session.on('input_speech_stopped', (ev: any) => {\n        this.emit('user_stopped_speaking');\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_started', (ev: any) => {\n        this.#pendingFunctionCalls.add(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_completed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      this.#session.on('function_call_failed', (ev: any) => {\n        this.#pendingFunctionCalls.delete(ev.callId);\n        this.#updateState();\n      });\n\n      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {\n        this.emit('metrics_collected', metrics);\n      });\n\n      resolve(this.#session);\n    });\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.linkedParticipant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    if (this.linkedParticipant.trackPublications.size > 0) {\n      this.#subscribeToMicrophone();\n    }\n\n    // also check if already subscribed\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {\n        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);\n        break;\n      }\n    }\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.linkedParticipant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.linkedParticipant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n  }\n\n  #handleTrackSubscription(\n    track: RemoteTrack,\n    publication: RemoteTrackPublication,\n    participant: RemoteParticipant,\n  ) {\n    if (\n      publication.source !== TrackSource.SOURCE_MICROPHONE ||\n      participant.identity !== this.linkedParticipant?.identity\n    ) {\n      return;\n    }\n    const readAudioStreamTask = async (audioStream: AudioStream) => {\n      const bstream = new AudioByteStream(\n        this.model.sampleRate,\n        this.model.numChannels,\n        this.model.inFrameSize,\n      );\n\n      for await (const frame of audioStream) {\n        const audioData = frame.data;\n        for (const frame of bstream.write(audioData.buffer)) {\n          this.#session!.inputAudioBuffer.append(frame);\n        }\n      }\n    };\n    this.subscribedTrack = track;\n\n    this.readMicroTask = new Promise<void>((resolve, reject) => {\n      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))\n        .then(resolve)\n        .catch(reject);\n    });\n  }\n\n  #getLocalTrackSid(): string | null {\n    if (!this.#localTrackSid && this.room && this.room.localParticipant) {\n      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant!.identity!);\n    }\n    return this.#localTrackSid;\n  }\n\n  #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ): void {\n    this.#logger.debug(\n      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,\n    );\n    if (!this.room?.localParticipant) {\n      this.#logger.error('Room or local participant not set');\n      return;\n    }\n\n    this.room.localParticipant.publishTranscription({\n      participantIdentity,\n      trackSid,\n      segments: [\n        {\n          text,\n          final: isFinal,\n          id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n  }\n\n  #updateState() {\n    let newState: AgentState = 'initializing';\n    if (this.#pendingFunctionCalls.size > 0) {\n      newState = 'thinking';\n    } else if (this.#speaking) {\n      newState = 'speaking';\n    } else if (this.#started) {\n      newState = 'listening';\n    }\n\n    this.#setState(newState);\n  }\n\n  #setState(state: AgentState) {\n    if (this.room?.isConnected && this.room.localParticipant) {\n      const currentState = this.room.localParticipant.attributes![AGENT_STATE_ATTRIBUTE];\n      if (currentState !== state) {\n        this.room.localParticipant.setAttributes({\n          [AGENT_STATE_ATTRIBUTE]: state,\n        });\n        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);\n      }\n    }\n  }\n}\n"],"mappings":"AAWA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,oBAAoB;AAC7B,SAAS,uBAAuB;AAChC,YAAY,SAAS;AACrB,SAAS,WAAW;AAEpB,SAAS,mCAAmC;AAC5C,SAAS,wBAAwB;AACjC,SAAS,oBAAwC;AAM1C,MAAe,wBAAwB,aAAa;AAO3D;AAMO,MAAe,cAAc;AAQpC;AAGO,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB,aAAa;AAAA,EAChD;AAAA,EACA,OAAoB;AAAA,EACpB,oBAA8C;AAAA,EAC9C,kBAA2C;AAAA,EAC3C,gBAAsC;AAAA,EAEtC,uBAAuB;AAAA,EACvB;AAAA,EAEA,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA,yBAAyB;AAAA,EAC3B,GAKG;AACD,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,eAAkD;AAAA,EAClD,oBAAkD;AAAA,EAClD,iBAAgC;AAAA,EAChC,eAAmC;AAAA,EACnC,gBAAqC;AAAA,EACrC,iBAA4C;AAAA,EAC5C,UAAU,IAAI;AAAA,EACd,WAAmC;AAAA,EACnC,UAA2C;AAAA,EAC3C,WAAwC;AAAA,EAExC,YAAqB;AAAA,EACrB,yBAAsC,oBAAI,IAAI;AAAA,EAC9C,aAAsB;AAAA,EAEtB,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AACf,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,SAAS;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,wBAAqC;AACvC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAAsB,OAAoB;AAC5C,SAAK,yBAAyB;AAC9B,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAU,YAAqB;AACjC,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,SAAkB;AAC7B,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MACE,MACA,cAAiD,MACvB;AAC1B,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAhJlD;AAiJM,UAAI,KAAK,UAAU;AACjB,eAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,MACrD;AACA,WAAK,aAAa;AAElB,WAAK,GAAG,UAAU,sBAAsB,CAACA,iBAAmC;AAE1E,YAAI,KAAK,mBAAmB;AAC1B;AAAA,QACF;AACA,aAAK,iBAAiBA,aAAY,QAAS;AAAA,MAC7C,CAAC;AACD,WAAK;AAAA,QACH,UAAU;AAAA,QACV,CAAC,kBAA0CA,iBAAmC;AAC5E,cACE,KAAK,qBACLA,aAAY,aAAa,KAAK,kBAAkB,YAChD,iBAAiB,WAAW,YAAY,qBACxC,CAAC,iBAAiB,YAClB;AACA,6BAAiB,cAAc,IAAI;AAAA,UACrC;AAAA,QACF;AAAA,MACF;AACA,WAAK,GAAG,UAAU,iBAAiB,KAAK,yBAAyB,KAAK,IAAI,CAAC;AAE3E,WAAK,OAAO;AACZ,WAAK,eAAe;AAEpB,WAAK,eAAe,IAAI,YAAY,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW;AACjF,WAAK,gBAAgB,IAAI;AAAA,QACvB,KAAK;AAAA,QACL,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AACA,YAAM,mBAAmB,MAAM;AAC7B,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AAAA,MACnB;AAEA,YAAM,mBAAmB,CAAC,gBAAyB;AACjD,aAAK,KAAK,wBAAwB;AAClC,aAAK,YAAY;AACjB,YAAI,KAAK,gBAAgB;AACvB,cAAI,OAAO,KAAK,eAAe,iBAAiB;AAChD,cAAI,aAAa;AACf,oBAAQ;AAAA,UACV;AACA,gBAAM,MAAM,IAAI,YAAY,OAAO;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB;AAAA,UACF,CAAC;AAED,cAAI,aAAa;AACf,iBAAK,KAAK,4BAA4B,GAAG;AAAA,UAC3C,OAAO;AACL,iBAAK,KAAK,0BAA0B,GAAG;AAAA,UACzC;AACA,eAAK,QAAQ,MAAM,EAAE,eAAe,MAAM,YAAY,CAAC,EAAE,MAAM,wBAAwB;AAAA,QACzF;AAAA,MACF;AAEA,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AACzD,WAAK,cAAc,GAAG,mBAAmB,gBAAgB;AAEzD,YAAM,QAAQ,gBAAgB,iBAAiB,mBAAmB,KAAK,YAAY;AACnF,YAAM,UAAU,IAAI,oBAAoB;AACxC,cAAQ,SAAS,YAAY;AAC7B,WAAK,oBAAqB,QAAM,UAAK,qBAAL,mBAAuB,aAAa,OAAO,aAAa;AACxF,UAAI,CAAC,KAAK,mBAAmB;AAC3B,aAAK,QAAQ,MAAM,yBAAyB;AAC5C,eAAO,IAAI,MAAM,yBAAyB,CAAC;AAC3C;AAAA,MACF;AAEA,YAAM,KAAK,kBAAkB,oBAAoB;AAEjD,UAAI,aAAa;AACf,YAAI,OAAO,gBAAgB,UAAU;AACnC,eAAK,iBAAiB,WAAW;AAAA,QACnC,OAAO;AACL,eAAK,iBAAiB,YAAY,QAAS;AAAA,QAC7C;AAAA,MACF,OAAO;AAEL,mBAAWA,gBAAe,KAAK,mBAAmB,OAAO,GAAG;AAC1D,eAAK,iBAAiBA,aAAY,QAAS;AAC3C;AAAA,QACF;AAAA,MACF;AAEA,WAAK,WAAW,KAAK,MAAM,QAAQ,EAAE,QAAQ,KAAK,SAAS,SAAS,KAAK,SAAS,CAAC;AACnF,WAAK,WAAW;AAGhB,WAAK,SAAS,GAAG,0BAA0B,CAAC,YAAiB;AAnPnE,YAAAC;AAqPQ,YAAI,QAAQ,gBAAgB,OAAQ;AAEpC,cAAM,QAAQ,IAAI;AAAA,UAChB,KAAK;AAAA,UACL,KAAK,KAAM,iBAAkB;AAAA,UAC7B,KAAK,kBAAkB;AAAA,UACvB,QAAQ;AAAA,QACV;AAEA,cAAM,UAASA,MAAA,KAAK,kBAAL,gBAAAA,IAAoB;AAAA,UACjC,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ;AAAA;AAEV,aAAK,iBAAiB;AAAA,MACxB,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,YAAiB;AAE1D,YAAI,QAAQ,gBAAgB,QAAQ;AAClC,cAAI,KAAK,wBAAwB,KAAK,yBAAyB;AAC7D,kBAAM,IAAI;AAAA,cACR,0DACW,KAAK,uBAAuB;AAAA,YAGzC;AAAA,UACF;AAEA,eAAK;AACL,eAAK,QACF,MAAM;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,MAAM,QAAQ;AAAA,YACd,SAAS,KAAK;AAAA,UAChB,CAAC,EACA;AAAA,YACC;AAAA,UAEF;AACF,eAAK,SAAU,wBAAwB,QAAQ,MAAM;AAAA,QACvD,OAAO;AACL,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,0BAA0B,CAAC,OAAY;AAvS9D,YAAAA,KAAA;AAySQ,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wCAAwC,CAAC,OAAY;AAnT5E,YAAAA,KAAA;AAqTQ,cAAM,gBAAgB,GAAG;AACzB,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,eAAe,MAAM,GAAG,MAAM;AAAA,QAC1F,OAAO;AACL,eAAK,QAAQ,MAAM,8BAA8B;AAAA,QACnD;AACA,cAAM,UAAU,IAAI,YAAY,OAAO;AAAA,UACrC,MAAM,IAAI,SAAS;AAAA,UACnB,MAAM;AAAA,QACR,CAAC;AACD,aAAK,KAAK,yBAAyB,OAAO;AAC1C,aAAK,QAAQ,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,uBAAuB;AAAA,MACrE,CAAC;AAED,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AArU5D,YAAAA,KAAA;AAsUQ,aAAK,KAAK,uBAAuB;AACjC,YAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,MAAM;AACpD,eAAK,eAAe,UAAU;AAE9B,eAAK,SAAU,aAAa,KAAK;AAAA,YAC/B,KAAK,eAAe;AAAA,YACpB,KAAK,eAAe;AAAA,YACpB,KAAK,MAAO,KAAK,eAAe,eAAe,OAAS,GAAI;AAAA,UAC9D;AAEA,eAAK,iBAAiB;AAAA,QACxB;AAEA,cAAM,uBAAsBA,MAAA,KAAK,sBAAL,gBAAAA,IAAwB;AACpD,cAAM,YAAW,UAAK,oBAAL,mBAAsB;AACvC,YAAI,uBAAuB,UAAU;AACnC,eAAK,sBAAsB,qBAAqB,UAAU,UAAK,OAAO,GAAG,MAAM;AAAA,QACjF;AAAA,MACF,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,KAAK,uBAAuB;AAAA,MACnC,CAAC;AAGD,WAAK,SAAS,GAAG,yBAAyB,CAAC,OAAY;AACrD,aAAK,sBAAsB,IAAI,GAAG,MAAM;AACxC,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,2BAA2B,CAAC,OAAY;AACvD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAGD,WAAK,SAAS,GAAG,wBAAwB,CAAC,OAAY;AACpD,aAAK,sBAAsB,OAAO,GAAG,MAAM;AAC3C,aAAK,aAAa;AAAA,MACpB,CAAC;AAED,WAAK,SAAS,GAAG,qBAAqB,CAAC,YAAkC;AACvE,aAAK,KAAK,qBAAqB,OAAO;AAAA,MACxC,CAAC;AAED,cAAQ,KAAK,QAAQ;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,MAAM;AACd,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,oBAAoB,KAAK,KAAK,mBAAmB,IAAI,mBAAmB,KAAK;AAClF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AACrD,WAAK,uBAAuB;AAAA,IAC9B;AAGA,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,YAAY,qBAAqB,YAAY,OAAO;AAC7E,aAAK,yBAAyB,YAAY,OAAO,aAAa,KAAK,iBAAiB;AACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,kBAAkB,kBAAkB,OAAO,GAAG;AAC3E,UAAI,YAAY,WAAW,YAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,yBACE,OACA,aACA,aACA;AA5aJ;AA6aI,QACE,YAAY,WAAW,YAAY,qBACnC,YAAY,eAAa,UAAK,sBAAL,mBAAwB,WACjD;AACA;AAAA,IACF;AACA,UAAM,sBAAsB,OAAO,gBAA6B;AAC9D,YAAM,UAAU,IAAI;AAAA,QAClB,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,QACX,KAAK,MAAM;AAAA,MACb;AAEA,uBAAiB,SAAS,aAAa;AACrC,cAAM,YAAY,MAAM;AACxB,mBAAWC,UAAS,QAAQ,MAAM,UAAU,MAAM,GAAG;AACnD,eAAK,SAAU,iBAAiB,OAAOA,MAAK;AAAA,QAC9C;AAAA,MACF;AAAA,IACF;AACA,SAAK,kBAAkB;AAEvB,SAAK,gBAAgB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC1D,0BAAoB,IAAI,YAAY,OAAO,KAAK,MAAM,YAAY,KAAK,MAAM,WAAW,CAAC,EACtF,KAAK,OAAO,EACZ,MAAM,MAAM;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,oBAAmC;AACjC,QAAI,CAAC,KAAK,kBAAkB,KAAK,QAAQ,KAAK,KAAK,kBAAkB;AACnE,WAAK,iBAAiB,iBAAiB,KAAK,MAAM,KAAK,KAAK,iBAAkB,QAAS;AAAA,IACzF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,sBACE,qBACA,UACA,MACA,SACA,IACM;AAvdV;AAwdI,SAAK,QAAQ;AAAA,MACX,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE;AAAA,IACtF;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,mBAAkB;AAChC,WAAK,QAAQ,MAAM,mCAAmC;AACtD;AAAA,IACF;AAEA,SAAK,KAAK,iBAAiB,qBAAqB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,eAAe;AACb,QAAI,WAAuB;AAC3B,QAAI,KAAK,sBAAsB,OAAO,GAAG;AACvC,iBAAW;AAAA,IACb,WAAW,KAAK,WAAW;AACzB,iBAAW;AAAA,IACb,WAAW,KAAK,UAAU;AACxB,iBAAW;AAAA,IACb;AAEA,SAAK,UAAU,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAU,OAAmB;AA7f/B;AA8fI,UAAI,UAAK,SAAL,mBAAW,gBAAe,KAAK,KAAK,kBAAkB;AACxD,YAAM,eAAe,KAAK,KAAK,iBAAiB,WAAY,qBAAqB;AACjF,UAAI,iBAAiB,OAAO;AAC1B,aAAK,KAAK,iBAAiB,cAAc;AAAA,UACvC,CAAC,qBAAqB,GAAG;AAAA,QAC3B,CAAC;AACD,aAAK,QAAQ,MAAM,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACF;","names":["participant","_a","frame"]}

package/dist/pipeline/pipeline_agent.cjs

@@ -323,7 +323,7 @@ class VoicePipelineAgent extends import_node_events.default {
       }
     });
     this.#humanInput.on(import_human_input.HumanInputEvent.END_OF_SPEECH, (event) => {
-      this.emit(0 /* USER_STARTED_SPEAKING */);
+      this.emit(1 /* USER_STOPPED_SPEAKING */);
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
     this.#humanInput.on(import_human_input.HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
@@ -683,6 +683,7 @@ class DeferredReplyValidation {
   #speaking = false;
   #endOfSpeechDelay;
   #finalTranscriptDelay;
+  #abort;
   constructor(validateFunc, minEndpointingDelay) {
     this.#validateFunc = validateFunc;
     this.#endOfSpeechDelay = minEndpointingDelay;
@@ -701,14 +702,18 @@ class DeferredReplyValidation {
   }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   onHumanStartOfSpeech(_) {
+    var _a;
     this.#speaking = true;
+    if (this.validating) {
+      (_a = this.#abort) == null ? void 0 : _a.abort();
+    }
   }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   onHumanEndOfSpeech(_) {
     this.#speaking = false;
     this.#lastRecvEndOfSpeechTime = Date.now();
     if (this.#lastFinalTranscript) {
-      const delay = this.#endWithPunctuation() ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR : 1;
+      const delay = this.#endWithPunctuation() ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR : 1e3;
       this.#run(delay);
     }
   }
@@ -721,13 +726,20 @@ class DeferredReplyValidation {
     this.#lastRecvEndOfSpeechTime = 0;
   }
   #run(delay) {
-    const runTask = async (delay2) => {
-      await new Promise((resolve) => setTimeout(resolve, delay2));
-      this.#resetStates();
-      await this.#validateFunc();
+    var _a;
+    const runTask = async (delay2, signal) => {
+      const timeout = setTimeout(() => {
+        this.#resetStates();
+        this.#validateFunc();
+      }, delay2);
+      signal.addEventListener("abort", () => {
+        clearTimeout(timeout);
+      });
     };
+    (_a = this.#abort) == null ? void 0 : _a.abort();
+    this.#abort = new AbortController();
     this.#validatingFuture = new import_utils.Future();
-    this.#validatingPromise = runTask(delay);
+    this.#validatingPromise = runTask(delay, this.#abort.signal);
   }
 }
 // Annotate the CommonJS export names for ESM import in node: