@livekit/agents 1.0.27 → 1.0.30

package/dist/voice/background_audio.cjs

@@ -74,8 +74,11 @@ class PlayHandle {
 class BackgroundAudioPlayer {
   ambientSound;
   thinkingSound;
+  streamTimeoutMs;
   playTasks = [];
   audioSource = new import_rtc_node.AudioSource(48e3, 1, AUDIO_SOURCE_BUFFER_MS);
+  audioMixer;
+  mixerTask;
   room;
   agentSession;
   publication;
@@ -83,15 +86,20 @@ class BackgroundAudioPlayer {
   republishTask;
   ambientHandle;
   thinkingHandle;
+  closed = true;
   // TODO (Brian): add lock
   #logger = (0, import_log.log)();
   constructor(options) {
-    const { ambientSound, thinkingSound } = options || {};
+    const { ambientSound, thinkingSound, streamTimeoutMs = 200 } = options || {};
     this.ambientSound = ambientSound;
     this.thinkingSound = thinkingSound;
-    if (this.thinkingSound) {
-      this.#logger.warn("thinkingSound is not yet supported");
-    }
+    this.streamTimeoutMs = streamTimeoutMs;
+    this.audioMixer = new import_rtc_node.AudioMixer(48e3, 1, {
+      blocksize: 4800,
+      // 100ms at 48kHz
+      capacity: 1,
+      streamTimeoutMs: this.streamTimeoutMs
+    });
   }
   /**
    * Select a sound from a list of background sound based on probability weights
@@ -190,7 +198,16 @@ class BackgroundAudioPlayer {
     this.room = room;
     this.agentSession = agentSession;
     this.trackPublishOptions = trackPublishOptions;
+    this.closed = false;
     await this.publishTrack();
+    this.mixerTask = import_utils.Task.from(async () => {
+      try {
+        await this.runMixerTask();
+      } catch (err) {
+        if (this.closed) return;
+        throw err;
+      }
+    });
     this.room.on("reconnected", this.onReconnected);
     (_a = this.agentSession) == null ? void 0 : _a.on(import_events.AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
     if (!this.ambientSound) return;
@@ -205,11 +222,16 @@ class BackgroundAudioPlayer {
    */
   async close() {
     var _a, _b, _c, _d;
+    this.closed = true;
     await (0, import_utils.cancelAndWait)(this.playTasks, TASK_TIMEOUT_MS);
     if (this.republishTask) {
       await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);
     }
+    await this.audioMixer.aclose();
     await this.audioSource.close();
+    if (this.mixerTask) {
+      await this.mixerTask.cancelAndWait(TASK_TIMEOUT_MS);
+    }
     (_a = this.agentSession) == null ? void 0 : _a.off(import_events.AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
     (_b = this.room) == null ? void 0 : _b.off("reconnected", this.onReconnected);
     if (this.publication && this.publication.sid) {
@@ -250,6 +272,11 @@ class BackgroundAudioPlayer {
   async republishTrackTask() {
     await this.publishTrack();
   }
+  async runMixerTask() {
+    for await (const frame of this.audioMixer) {
+      await this.audioSource.captureFrame(frame);
+    }
+  }
   onAgentStateChanged = (ev) => {
     var _a;
     if (!this.thinkingSound) {
@@ -259,10 +286,38 @@ class BackgroundAudioPlayer {
       if (this.thinkingHandle && !this.thinkingHandle.done()) {
         return;
       }
+      const normalized = this.normalizeSoundSource(this.thinkingSound);
+      if (normalized) {
+        const { source, volume } = normalized;
+        const selectedSound = { source, volume, probability: 1 };
+        this.thinkingHandle = this.play(selectedSound, typeof source === "string");
+      }
     } else {
       (_a = this.thinkingHandle) == null ? void 0 : _a.stop();
     }
   };
+  // Note: Python uses numpy, TS uses typed arrays for equivalent logic
+  applyVolumeToFrame(frame, volume) {
+    const int16Data = new Int16Array(
+      frame.data.buffer,
+      frame.data.byteOffset,
+      frame.data.byteLength / 2
+    );
+    const float32Data = new Float32Array(int16Data.length);
+    for (let i = 0; i < int16Data.length; i++) {
+      float32Data[i] = int16Data[i];
+    }
+    const volumeFactor = 10 ** Math.log10(volume);
+    for (let i = 0; i < float32Data.length; i++) {
+      float32Data[i] *= volumeFactor;
+    }
+    const outputData = new Int16Array(float32Data.length);
+    for (let i = 0; i < float32Data.length; i++) {
+      const clipped = Math.max(-32768, Math.min(32767, float32Data[i]));
+      outputData[i] = Math.round(clipped);
+    }
+    return new import_rtc_node.AudioFrame(outputData, frame.sampleRate, frame.channels, frame.samplesPerChannel);
+  }
   async playTask({
     playHandle,
     sound,
@@ -273,45 +328,30 @@ class BackgroundAudioPlayer {
     if (isBuiltinAudioClip(sound)) {
       sound = getBuiltinAudioPath(sound);
     }
+    let audioStream;
     if (typeof sound === "string") {
-      sound = loop ? (0, import_audio.loopAudioFramesFromFile)(sound, { abortSignal: signal }) : (0, import_audio.audioFramesFromFile)(sound, { abortSignal: signal });
+      audioStream = loop ? (0, import_audio.loopAudioFramesFromFile)(sound, { abortSignal: signal }) : (0, import_audio.audioFramesFromFile)(sound, { abortSignal: signal });
+    } else {
+      audioStream = sound;
     }
-    try {
-      for await (const frame of sound) {
+    const applyVolume = this.applyVolumeToFrame.bind(this);
+    async function* genWrapper() {
+      for await (const frame of audioStream) {
         if (signal.aborted || playHandle.done()) break;
-        let processedFrame;
-        if (volume !== 1) {
-          const int16Data = new Int16Array(
-            frame.data.buffer,
-            frame.data.byteOffset,
-            frame.data.byteLength / 2
-          );
-          const float32Data = new Float32Array(int16Data.length);
-          for (let i = 0; i < int16Data.length; i++) {
-            float32Data[i] = int16Data[i];
-          }
-          const volumeFactor = 10 ** Math.log10(volume);
-          for (let i = 0; i < float32Data.length; i++) {
-            float32Data[i] *= volumeFactor;
-          }
-          const outputData = new Int16Array(float32Data.length);
-          for (let i = 0; i < float32Data.length; i++) {
-            const clipped = Math.max(-32768, Math.min(32767, float32Data[i]));
-            outputData[i] = Math.round(clipped);
-          }
-          processedFrame = new import_rtc_node.AudioFrame(
-            outputData,
-            frame.sampleRate,
-            frame.channels,
-            frame.samplesPerChannel
-          );
-        } else {
-          processedFrame = frame;
-        }
-        await this.audioSource.captureFrame(processedFrame);
+        yield volume !== 1 ? applyVolume(frame, volume) : frame;
       }
+      playHandle._markPlayoutDone();
+    }
+    const gen = genWrapper();
+    try {
+      this.audioMixer.addStream(gen);
+      await playHandle.waitForPlayout();
     } finally {
+      this.audioMixer.removeStream(gen);
       playHandle._markPlayoutDone();
+      if (playHandle.done()) {
+        await gen.return(void 0);
+      }
     }
   }
 }

package/dist/voice/background_audio.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/background_audio.ts","../../../node_modules/.pnpm/tsup@8.4.0_@microsoft+api-extractor@7.43.7_@types+node@22.15.30__postcss@8.4.38_tsx@4.20.4_typescript@5.4.5/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AudioFrame,\n  AudioSource,\n  LocalAudioTrack,\n  type LocalTrackPublication,\n  type Room,\n  TrackPublishOptions,\n} from '@livekit/rtc-node';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { audioFramesFromFile, loopAudioFramesFromFile } from '../audio.js';\nimport { log } from '../log.js';\nimport { Future, Task, cancelAndWait } from '../utils.js';\nimport type { AgentSession } from './agent_session.js';\nimport { AgentSessionEventTypes, type AgentStateChangedEvent } from './events.js';\n\nconst TASK_TIMEOUT_MS = 500;\n\nexport enum BuiltinAudioClip {\n  OFFICE_AMBIENCE = 'office-ambience.ogg',\n  KEYBOARD_TYPING = 'keyboard-typing.ogg',\n  KEYBOARD_TYPING2 = 'keyboard-typing2.ogg',\n}\n\nexport function isBuiltinAudioClip(\n  source: AudioSourceType | AudioConfig | AudioConfig[],\n): source is BuiltinAudioClip {\n  return (\n    typeof source === 'string' &&\n    Object.values(BuiltinAudioClip).includes(source as BuiltinAudioClip)\n  );\n}\n\nexport function getBuiltinAudioPath(clip: BuiltinAudioClip): string {\n  const resourcesPath = join(dirname(fileURLToPath(import.meta.url)), '../../resources');\n  return join(resourcesPath, clip);\n}\n\nexport type AudioSourceType = string | BuiltinAudioClip | AsyncIterable<AudioFrame>;\n\nexport interface AudioConfig {\n  source: AudioSourceType;\n  volume?: number;\n  probability?: number;\n}\n\nexport interface BackgroundAudioPlayerOptions {\n  /**\n   * Ambient sound to play continuously in the background.\n   * Can be a file path, BuiltinAudioClip, or AudioConfig.\n   * File paths will be looped automatically.\n   */\n  ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  /**\n   * Sound to play when the agent is thinking.\n   * TODO (Brian): Implement thinking sound when AudioMixer becomes available\n   */\n  thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  /**\n   * Stream timeout in milliseconds\n   * @defaultValue 200\n   */\n  streamTimeoutMs?: number;\n}\n\nexport interface BackgroundAudioStartOptions {\n  room: Room;\n  agentSession?: AgentSession;\n  trackPublishOptions?: TrackPublishOptions;\n}\n\n// Queue size for AudioSource buffer (400ms)\n// Kept small to avoid abrupt cutoffs when removing sounds\nconst AUDIO_SOURCE_BUFFER_MS = 400;\n\nexport class PlayHandle {\n  private doneFuture = new Future<void>();\n  private stopFuture = new Future<void>();\n\n  done(): boolean {\n    return this.doneFuture.done;\n  }\n\n  stop(): void {\n    if (this.done()) return;\n\n    if (!this.stopFuture.done) {\n      this.stopFuture.resolve();\n    }\n\n    this._markPlayoutDone();\n  }\n\n  async waitForPlayout(): Promise<void> {\n    return this.doneFuture.await;\n  }\n\n  _markPlayoutDone(): void {\n    if (!this.doneFuture.done) {\n      this.doneFuture.resolve();\n    }\n  }\n}\n\n/**\n * Manages background audio playback for LiveKit agent sessions\n *\n * This class handles playing ambient sounds and manages audio track publishing.\n * It supports:\n * - Continuous ambient sound playback with looping\n * - Volume control and probability-based sound selection\n * - Integration with LiveKit rooms and agent sessions\n *\n * Note: Thinking sound not yet supported\n *\n * @example\n * ```typescript\n * const player = new BackgroundAudioPlayer({\n *   ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },\n * });\n *\n * await player.start({ room, agentSession });\n * ```\n */\nexport class BackgroundAudioPlayer {\n  private ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n  private thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  private playTasks: Task<void>[] = [];\n  private audioSource = new AudioSource(48000, 1, AUDIO_SOURCE_BUFFER_MS);\n\n  private room?: Room;\n  private agentSession?: AgentSession;\n  private publication?: LocalTrackPublication;\n  private trackPublishOptions?: TrackPublishOptions;\n  private republishTask?: Task<void>;\n\n  private ambientHandle?: PlayHandle;\n  private thinkingHandle?: PlayHandle;\n\n  // TODO (Brian): add lock\n\n  #logger = log();\n\n  constructor(options?: BackgroundAudioPlayerOptions) {\n    const { ambientSound, thinkingSound } = options || {};\n\n    this.ambientSound = ambientSound;\n    this.thinkingSound = thinkingSound;\n\n    if (this.thinkingSound) {\n      this.#logger.warn('thinkingSound is not yet supported');\n      // TODO: Implement thinking sound when AudioMixer becomes available\n    }\n  }\n\n  /**\n   * Select a sound from a list of background sound based on probability weights\n   * Return undefined if no sound is selected (when sum of probabilities < 1.0).\n   */\n  private selectSoundFromList(sounds: AudioConfig[]): AudioConfig | undefined {\n    const totalProbability = sounds.reduce((sum, sound) => sum + (sound.probability ?? 1.0), 0);\n\n    if (totalProbability <= 0) {\n      return undefined;\n    }\n\n    if (totalProbability < 1.0 && Math.random() > totalProbability) {\n      return undefined;\n    }\n\n    const normalizeFactor = totalProbability <= 1.0 ? 1.0 : totalProbability;\n    const r = Math.random() * Math.min(totalProbability, 1.0);\n    let cumulative = 0.0;\n\n    for (const sound of sounds) {\n      const prob = sound.probability ?? 1.0;\n      if (prob <= 0) {\n        continue;\n      }\n\n      const normProb = prob / normalizeFactor;\n      cumulative += normProb;\n\n      if (r <= cumulative) {\n        return sound;\n      }\n    }\n\n    return sounds[sounds.length - 1];\n  }\n\n  private normalizeSoundSource(\n    source?: AudioSourceType | AudioConfig | AudioConfig[],\n  ): { source: AudioSourceType; volume: number } | undefined {\n    if (source === undefined) {\n      return undefined;\n    }\n\n    if (typeof source === 'string') {\n      return {\n        source: this.normalizeBuiltinAudio(source),\n        volume: 1.0,\n      };\n    }\n\n    if (Array.isArray(source)) {\n      const selected = this.selectSoundFromList(source);\n      if (selected === undefined) {\n        return undefined;\n      }\n\n      return {\n        source: selected.source,\n        volume: selected.volume ?? 1.0,\n      };\n    }\n\n    if (typeof source === 'object' && 'source' in source) {\n      return {\n        source: this.normalizeBuiltinAudio(source.source),\n        volume: source.volume ?? 1.0,\n      };\n    }\n\n    return { source, volume: 1.0 };\n  }\n\n  private normalizeBuiltinAudio(source: AudioSourceType): AudioSourceType {\n    if (isBuiltinAudioClip(source)) {\n      return getBuiltinAudioPath(source);\n    }\n    return source;\n  }\n\n  play(audio: AudioSourceType | AudioConfig | AudioConfig[], loop = false): PlayHandle {\n    const normalized = this.normalizeSoundSource(audio);\n    if (normalized === undefined) {\n      const handle = new PlayHandle();\n      handle._markPlayoutDone();\n      return handle;\n    }\n\n    const { source, volume } = normalized;\n    const playHandle = new PlayHandle();\n\n    const task = Task.from(async ({ signal }) => {\n      await this.playTask({ playHandle, sound: source, volume, loop, signal });\n    });\n\n    task.addDoneCallback(() => {\n      playHandle._markPlayoutDone();\n      this.playTasks.splice(this.playTasks.indexOf(task), 1);\n    });\n\n    this.playTasks.push(task);\n    return playHandle;\n  }\n\n  /**\n   * Start the background audio system, publishing the audio track\n   * and beginning playback of any configured ambient sound.\n   *\n   * If `ambientSound` is provided (and contains file paths), they will loop\n   * automatically. If `ambientSound` contains AsyncIterators, they are assumed\n   * to be already infinite or looped.\n   *\n   * @param options - Options for starting background audio playback\n   */\n  async start(options: BackgroundAudioStartOptions): Promise<void> {\n    const { room, agentSession, trackPublishOptions } = options;\n    this.room = room;\n    this.agentSession = agentSession;\n    this.trackPublishOptions = trackPublishOptions;\n\n    await this.publishTrack();\n\n    // TODO (Brian): check job context is not fake\n\n    // TODO (Brian): start audio mixer task\n    this.room.on('reconnected', this.onReconnected);\n\n    this.agentSession?.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n\n    if (!this.ambientSound) return;\n\n    const normalized = this.normalizeSoundSource(this.ambientSound);\n    if (!normalized) return;\n\n    const { source, volume } = normalized;\n    const selectedSound: AudioConfig = { source, volume, probability: 1.0 };\n    this.ambientHandle = this.play(selectedSound, typeof source === 'string');\n  }\n\n  /**\n   * Close and cleanup the background audio system\n   */\n  async close(): Promise<void> {\n    await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);\n\n    if (this.republishTask) {\n      await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);\n    }\n\n    // TODO (Brian): cancel audio mixer task and close audio mixer\n\n    await this.audioSource.close();\n\n    this.agentSession?.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n    this.room?.off('reconnected', this.onReconnected);\n\n    if (this.publication && this.publication.sid) {\n      await this.room?.localParticipant?.unpublishTrack(this.publication.sid);\n    }\n  }\n\n  /**\n   * Get the current track publication\n   */\n  getPublication(): LocalTrackPublication | undefined {\n    return this.publication;\n  }\n\n  private async publishTrack(): Promise<void> {\n    if (this.publication !== undefined) {\n      return;\n    }\n\n    const track = LocalAudioTrack.createAudioTrack('background_audio', this.audioSource);\n\n    if (this.room?.localParticipant === undefined) {\n      throw new Error('Local participant not available');\n    }\n\n    const publication = await this.room.localParticipant.publishTrack(\n      track,\n      this.trackPublishOptions ?? new TrackPublishOptions(),\n    );\n\n    this.publication = publication;\n    this.#logger.debug(`Background audio track published: ${this.publication.sid}`);\n  }\n\n  private onReconnected = (): void => {\n    if (this.republishTask) {\n      this.republishTask.cancel();\n    }\n\n    this.publication = undefined;\n    this.republishTask = Task.from(async () => {\n      await this.republishTrackTask();\n    });\n  };\n\n  private async republishTrackTask(): Promise<void> {\n    // TODO (Brian): add lock protection when implementing lock\n    await this.publishTrack();\n  }\n\n  private onAgentStateChanged = (ev: AgentStateChangedEvent): void => {\n    if (!this.thinkingSound) {\n      return;\n    }\n\n    if (ev.newState === 'thinking') {\n      if (this.thinkingHandle && !this.thinkingHandle.done()) {\n        return;\n      }\n\n      // TODO (Brian): play thinking sound and assign to thinkingHandle\n    } else {\n      this.thinkingHandle?.stop();\n    }\n  };\n\n  private async playTask({\n    playHandle,\n    sound,\n    volume,\n    loop,\n    signal,\n  }: {\n    playHandle: PlayHandle;\n    sound: AudioSourceType;\n    volume: number;\n    loop: boolean;\n    signal: AbortSignal;\n  }): Promise<void> {\n    if (isBuiltinAudioClip(sound)) {\n      sound = getBuiltinAudioPath(sound);\n    }\n\n    if (typeof sound === 'string') {\n      sound = loop\n        ? loopAudioFramesFromFile(sound, { abortSignal: signal })\n        : audioFramesFromFile(sound, { abortSignal: signal });\n    }\n\n    try {\n      for await (const frame of sound) {\n        if (signal.aborted || playHandle.done()) break;\n\n        let processedFrame: AudioFrame;\n\n        if (volume !== 1.0) {\n          const int16Data = new Int16Array(\n            frame.data.buffer,\n            frame.data.byteOffset,\n            frame.data.byteLength / 2,\n          );\n          const float32Data = new Float32Array(int16Data.length);\n\n          for (let i = 0; i < int16Data.length; i++) {\n            float32Data[i] = int16Data[i]!;\n          }\n\n          const volumeFactor = 10 ** Math.log10(volume);\n          for (let i = 0; i < float32Data.length; i++) {\n            float32Data[i]! *= volumeFactor;\n          }\n\n          const outputData = new Int16Array(float32Data.length);\n          for (let i = 0; i < float32Data.length; i++) {\n            const clipped = Math.max(-32768, Math.min(32767, float32Data[i]!));\n            outputData[i] = Math.round(clipped);\n          }\n\n          processedFrame = new AudioFrame(\n            outputData,\n            frame.sampleRate,\n            frame.channels,\n            frame.samplesPerChannel,\n          );\n        } else {\n          processedFrame = frame;\n        }\n\n        // TODO (Brian): use AudioMixer to add/remove frame streams\n        await this.audioSource.captureFrame(processedFrame);\n      }\n    } finally {\n      // TODO: the waitForPlayout() may be innaccurate by 400ms\n      playHandle._markPlayoutDone();\n    }\n  }\n}\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () =>\n  typeof document === 'undefined'\n    ? new URL(`file:${__filename}`).href\n    : (document.currentScript && document.currentScript.src) ||\n      new URL('main.js', document.baseURI).href\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,OAClD,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEpC,IAAM,gBAAgC,iCAAiB;ADR9D,sBAOO;AACP,uBAA8B;AAC9B,sBAA8B;AAC9B,mBAA6D;AAC7D,iBAAoB;AACpB,mBAA4C;AAE5C,oBAAoE;AAEpE,MAAM,kBAAkB;AAEjB,IAAK,mBAAL,kBAAKA,sBAAL;AACL,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,sBAAmB;AAHT,SAAAA;AAAA,GAAA;AAML,SAAS,mBACd,QAC4B;AAC5B,SACE,OAAO,WAAW,YAClB,OAAO,OAAO,gBAAgB,EAAE,SAAS,MAA0B;AAEvE;AAEO,SAAS,oBAAoB,MAAgC;AAClE,QAAM,oBAAgB,2BAAK,8BAAQ,+BAAc,aAAe,CAAC,GAAG,iBAAiB;AACrF,aAAO,uBAAK,eAAe,IAAI;AACjC;AAuCA,MAAM,yBAAyB;AAExB,MAAM,WAAW;AAAA,EACd,aAAa,IAAI,oBAAa;AAAA,EAC9B,aAAa,IAAI,oBAAa;AAAA,EAEtC,OAAgB;AACd,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,OAAa;AACX,QAAI,KAAK,KAAK,EAAG;AAEjB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,MAAM,iBAAgC;AACpC,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,mBAAyB;AACvB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAAA,EACF;AACF;AAsBO,MAAM,sBAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EAEA,YAA0B,CAAC;AAAA,EAC3B,cAAc,IAAI,4BAAY,MAAO,GAAG,sBAAsB;AAAA,EAE9D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAIR,cAAU,gBAAI;AAAA,EAEd,YAAY,SAAwC;AAClD,UAAM,EAAE,cAAc,cAAc,IAAI,WAAW,CAAC;AAEpD,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAErB,QAAI,KAAK,eAAe;AACtB,WAAK,QAAQ,KAAK,oCAAoC;AAAA,IAExD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,oBAAoB,QAAgD;AAC1E,UAAM,mBAAmB,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,MAAM,eAAe,IAAM,CAAC;AAE1F,QAAI,oBAAoB,GAAG;AACzB,aAAO;AAAA,IACT;AAEA,QAAI,mBAAmB,KAAO,KAAK,OAAO,IAAI,kBAAkB;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,oBAAoB,IAAM,IAAM;AACxD,UAAM,IAAI,KAAK,OAAO,IAAI,KAAK,IAAI,kBAAkB,CAAG;AACxD,QAAI,aAAa;AAEjB,eAAW,SAAS,QAAQ;AAC1B,YAAM,OAAO,MAAM,eAAe;AAClC,UAAI,QAAQ,GAAG;AACb;AAAA,MACF;AAEA,YAAM,WAAW,OAAO;AACxB,oBAAc;AAEd,UAAI,KAAK,YAAY;AACnB,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,OAAO,OAAO,SAAS,CAAC;AAAA,EACjC;AAAA,EAEQ,qBACN,QACyD;AACzD,QAAI,WAAW,QAAW;AACxB,aAAO;AAAA,IACT;AAEA,QAAI,OAAO,WAAW,UAAU;AAC9B,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,MAAM;AAAA,QACzC,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,YAAM,WAAW,KAAK,oBAAoB,MAAM;AAChD,UAAI,aAAa,QAAW;AAC1B,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL,QAAQ,SAAS;AAAA,QACjB,QAAQ,SAAS,UAAU;AAAA,MAC7B;AAAA,IACF;AAEA,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,OAAO,MAAM;AAAA,QAChD,QAAQ,OAAO,UAAU;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ,QAAQ,EAAI;AAAA,EAC/B;AAAA,EAEQ,sBAAsB,QAA0C;AACtE,QAAI,mBAAmB,MAAM,GAAG;AAC9B,aAAO,oBAAoB,MAAM;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,OAAsD,OAAO,OAAmB;AACnF,UAAM,aAAa,KAAK,qBAAqB,KAAK;AAClD,QAAI,eAAe,QAAW;AAC5B,YAAM,SAAS,IAAI,WAAW;AAC9B,aAAO,iBAAiB;AACxB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,aAAa,IAAI,WAAW;AAElC,UAAM,OAAO,kBAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AAC3C,YAAM,KAAK,SAAS,EAAE,YAAY,OAAO,QAAQ,QAAQ,MAAM,OAAO,CAAC;AAAA,IACzE,CAAC;AAED,SAAK,gBAAgB,MAAM;AACzB,iBAAW,iBAAiB;AAC5B,WAAK,UAAU,OAAO,KAAK,UAAU,QAAQ,IAAI,GAAG,CAAC;AAAA,IACvD,CAAC;AAED,SAAK,UAAU,KAAK,IAAI;AACxB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,MAAM,SAAqD;AAlRnE;AAmRI,UAAM,EAAE,MAAM,cAAc,oBAAoB,IAAI;AACpD,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB;AAE3B,UAAM,KAAK,aAAa;AAKxB,SAAK,KAAK,GAAG,eAAe,KAAK,aAAa;AAE9C,eAAK,iBAAL,mBAAmB,GAAG,qCAAuB,mBAAmB,KAAK;AAErE,QAAI,CAAC,KAAK,aAAc;AAExB,UAAM,aAAa,KAAK,qBAAqB,KAAK,YAAY;AAC9D,QAAI,CAAC,WAAY;AAEjB,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,gBAA6B,EAAE,QAAQ,QAAQ,aAAa,EAAI;AACtE,SAAK,gBAAgB,KAAK,KAAK,eAAe,OAAO,WAAW,QAAQ;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AA9S/B;AA+SI,cAAM,4BAAc,KAAK,WAAW,eAAe;AAEnD,QAAI,KAAK,eAAe;AACtB,YAAM,KAAK,cAAc,cAAc,eAAe;AAAA,IACxD;AAIA,UAAM,KAAK,YAAY,MAAM;AAE7B,eAAK,iBAAL,mBAAmB,IAAI,qCAAuB,mBAAmB,KAAK;AACtE,eAAK,SAAL,mBAAW,IAAI,eAAe,KAAK;AAEnC,QAAI,KAAK,eAAe,KAAK,YAAY,KAAK;AAC5C,cAAM,gBAAK,SAAL,mBAAW,qBAAX,mBAA6B,eAAe,KAAK,YAAY;AAAA,IACrE;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAoD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,eAA8B;AAxU9C;AAyUI,QAAI,KAAK,gBAAgB,QAAW;AAClC;AAAA,IACF;AAEA,UAAM,QAAQ,gCAAgB,iBAAiB,oBAAoB,KAAK,WAAW;AAEnF,UAAI,UAAK,SAAL,mBAAW,sBAAqB,QAAW;AAC7C,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,UAAM,cAAc,MAAM,KAAK,KAAK,iBAAiB;AAAA,MACnD;AAAA,MACA,KAAK,uBAAuB,IAAI,oCAAoB;AAAA,IACtD;AAEA,SAAK,cAAc;AACnB,SAAK,QAAQ,MAAM,qCAAqC,KAAK,YAAY,GAAG,EAAE;AAAA,EAChF;AAAA,EAEQ,gBAAgB,MAAY;AAClC,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,SAAK,cAAc;AACnB,SAAK,gBAAgB,kBAAK,KAAK,YAAY;AACzC,YAAM,KAAK,mBAAmB;AAAA,IAChC,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,qBAAoC;AAEhD,UAAM,KAAK,aAAa;AAAA,EAC1B;AAAA,EAEQ,sBAAsB,CAAC,OAAqC;AA5WtE;AA6WI,QAAI,CAAC,KAAK,eAAe;AACvB;AAAA,IACF;AAEA,QAAI,GAAG,aAAa,YAAY;AAC9B,UAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,KAAK,GAAG;AACtD;AAAA,MACF;AAAA,IAGF,OAAO;AACL,iBAAK,mBAAL,mBAAqB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAc,SAAS;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMkB;AAChB,QAAI,mBAAmB,KAAK,GAAG;AAC7B,cAAQ,oBAAoB,KAAK;AAAA,IACnC;AAEA,QAAI,OAAO,UAAU,UAAU;AAC7B,cAAQ,WACJ,sCAAwB,OAAO,EAAE,aAAa,OAAO,CAAC,QACtD,kCAAoB,OAAO,EAAE,aAAa,OAAO,CAAC;AAAA,IACxD;AAEA,QAAI;AACF,uBAAiB,SAAS,OAAO;AAC/B,YAAI,OAAO,WAAW,WAAW,KAAK,EAAG;AAEzC,YAAI;AAEJ,YAAI,WAAW,GAAK;AAClB,gBAAM,YAAY,IAAI;AAAA,YACpB,MAAM,KAAK;AAAA,YACX,MAAM,KAAK;AAAA,YACX,MAAM,KAAK,aAAa;AAAA,UAC1B;AACA,gBAAM,cAAc,IAAI,aAAa,UAAU,MAAM;AAErD,mBAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,wBAAY,CAAC,IAAI,UAAU,CAAC;AAAA,UAC9B;AAEA,gBAAM,eAAe,MAAM,KAAK,MAAM,MAAM;AAC5C,mBAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,wBAAY,CAAC,KAAM;AAAA,UACrB;AAEA,gBAAM,aAAa,IAAI,WAAW,YAAY,MAAM;AACpD,mBAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,kBAAM,UAAU,KAAK,IAAI,QAAQ,KAAK,IAAI,OAAO,YAAY,CAAC,CAAE,CAAC;AACjE,uBAAW,CAAC,IAAI,KAAK,MAAM,OAAO;AAAA,UACpC;AAEA,2BAAiB,IAAI;AAAA,YACnB;AAAA,YACA,MAAM;AAAA,YACN,MAAM;AAAA,YACN,MAAM;AAAA,UACR;AAAA,QACF,OAAO;AACL,2BAAiB;AAAA,QACnB;AAGA,cAAM,KAAK,YAAY,aAAa,cAAc;AAAA,MACpD;AAAA,IACF,UAAE;AAEA,iBAAW,iBAAiB;AAAA,IAC9B;AAAA,EACF;AACF;","names":["BuiltinAudioClip"]}
+{"version":3,"sources":["../../src/voice/background_audio.ts","../../../node_modules/.pnpm/tsup@8.4.0_@microsoft+api-extractor@7.43.7_@types+node@22.15.30__postcss@8.4.38_tsx@4.20.4_typescript@5.4.5/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AudioFrame,\n  AudioMixer,\n  AudioSource,\n  LocalAudioTrack,\n  type LocalTrackPublication,\n  type Room,\n  TrackPublishOptions,\n} from '@livekit/rtc-node';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { audioFramesFromFile, loopAudioFramesFromFile } from '../audio.js';\nimport { log } from '../log.js';\nimport { Future, Task, cancelAndWait } from '../utils.js';\nimport type { AgentSession } from './agent_session.js';\nimport { AgentSessionEventTypes, type AgentStateChangedEvent } from './events.js';\n\nconst TASK_TIMEOUT_MS = 500;\n\nexport enum BuiltinAudioClip {\n  OFFICE_AMBIENCE = 'office-ambience.ogg',\n  KEYBOARD_TYPING = 'keyboard-typing.ogg',\n  KEYBOARD_TYPING2 = 'keyboard-typing2.ogg',\n}\n\nexport function isBuiltinAudioClip(\n  source: AudioSourceType | AudioConfig | AudioConfig[],\n): source is BuiltinAudioClip {\n  return (\n    typeof source === 'string' &&\n    Object.values(BuiltinAudioClip).includes(source as BuiltinAudioClip)\n  );\n}\n\nexport function getBuiltinAudioPath(clip: BuiltinAudioClip): string {\n  const resourcesPath = join(dirname(fileURLToPath(import.meta.url)), '../../resources');\n  return join(resourcesPath, clip);\n}\n\nexport type AudioSourceType = string | BuiltinAudioClip | AsyncIterable<AudioFrame>;\n\nexport interface AudioConfig {\n  source: AudioSourceType;\n  volume?: number;\n  probability?: number;\n}\n\nexport interface BackgroundAudioPlayerOptions {\n  /**\n   * Ambient sound to play continuously in the background.\n   * Can be a file path, BuiltinAudioClip, or AudioConfig.\n   * File paths will be looped automatically.\n   */\n  ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  /**\n   * Sound to play when the agent is thinking.\n   * Plays when agent state changes to 'thinking' and stops when it changes to other states.\n   */\n  thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  /**\n   * Stream timeout in milliseconds\n   * @defaultValue 200\n   */\n  streamTimeoutMs?: number;\n}\n\nexport interface BackgroundAudioStartOptions {\n  room: Room;\n  agentSession?: AgentSession;\n  trackPublishOptions?: TrackPublishOptions;\n}\n\n// Queue size for AudioSource buffer (400ms)\n// Kept small to avoid abrupt cutoffs when removing sounds\nconst AUDIO_SOURCE_BUFFER_MS = 400;\n\nexport class PlayHandle {\n  private doneFuture = new Future<void>();\n  private stopFuture = new Future<void>();\n\n  done(): boolean {\n    return this.doneFuture.done;\n  }\n\n  stop(): void {\n    if (this.done()) return;\n\n    if (!this.stopFuture.done) {\n      this.stopFuture.resolve();\n    }\n\n    this._markPlayoutDone();\n  }\n\n  async waitForPlayout(): Promise<void> {\n    return this.doneFuture.await;\n  }\n\n  _markPlayoutDone(): void {\n    if (!this.doneFuture.done) {\n      this.doneFuture.resolve();\n    }\n  }\n}\n\n/**\n * Manages background audio playback for LiveKit agent sessions\n *\n * This class handles playing ambient sounds and manages audio track publishing.\n * It supports:\n * - Continuous ambient sound playback with looping\n * - Thinking sound playback during agent processing\n * - Multiple simultaneous audio streams via AudioMixer\n * - Volume control and probability-based sound selection\n * - Integration with LiveKit rooms and agent sessions\n *\n * @example\n * ```typescript\n * const player = new BackgroundAudioPlayer({\n *   ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },\n *   thinkingSound: { source: BuiltinAudioClip.KEYBOARD_TYPING, volume: 0.6 },\n * });\n *\n * await player.start({ room, agentSession });\n * ```\n */\nexport class BackgroundAudioPlayer {\n  private ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n  private thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n  private streamTimeoutMs: number;\n\n  private playTasks: Task<void>[] = [];\n  private audioSource = new AudioSource(48000, 1, AUDIO_SOURCE_BUFFER_MS);\n  private audioMixer: AudioMixer;\n  private mixerTask?: Task<void>;\n\n  private room?: Room;\n  private agentSession?: AgentSession;\n  private publication?: LocalTrackPublication;\n  private trackPublishOptions?: TrackPublishOptions;\n  private republishTask?: Task<void>;\n\n  private ambientHandle?: PlayHandle;\n  private thinkingHandle?: PlayHandle;\n\n  private closed = true;\n\n  // TODO (Brian): add lock\n\n  #logger = log();\n\n  constructor(options?: BackgroundAudioPlayerOptions) {\n    const { ambientSound, thinkingSound, streamTimeoutMs = 200 } = options || {};\n\n    this.ambientSound = ambientSound;\n    this.thinkingSound = thinkingSound;\n    this.streamTimeoutMs = streamTimeoutMs;\n\n    this.audioMixer = new AudioMixer(48000, 1, {\n      blocksize: 4800, // 100ms at 48kHz\n      capacity: 1,\n      streamTimeoutMs: this.streamTimeoutMs,\n    });\n  }\n\n  /**\n   * Select a sound from a list of background sound based on probability weights\n   * Return undefined if no sound is selected (when sum of probabilities < 1.0).\n   */\n  private selectSoundFromList(sounds: AudioConfig[]): AudioConfig | undefined {\n    const totalProbability = sounds.reduce((sum, sound) => sum + (sound.probability ?? 1.0), 0);\n\n    if (totalProbability <= 0) {\n      return undefined;\n    }\n\n    if (totalProbability < 1.0 && Math.random() > totalProbability) {\n      return undefined;\n    }\n\n    const normalizeFactor = totalProbability <= 1.0 ? 1.0 : totalProbability;\n    const r = Math.random() * Math.min(totalProbability, 1.0);\n    let cumulative = 0.0;\n\n    for (const sound of sounds) {\n      const prob = sound.probability ?? 1.0;\n      if (prob <= 0) {\n        continue;\n      }\n\n      const normProb = prob / normalizeFactor;\n      cumulative += normProb;\n\n      if (r <= cumulative) {\n        return sound;\n      }\n    }\n\n    return sounds[sounds.length - 1];\n  }\n\n  private normalizeSoundSource(\n    source?: AudioSourceType | AudioConfig | AudioConfig[],\n  ): { source: AudioSourceType; volume: number } | undefined {\n    if (source === undefined) {\n      return undefined;\n    }\n\n    if (typeof source === 'string') {\n      return {\n        source: this.normalizeBuiltinAudio(source),\n        volume: 1.0,\n      };\n    }\n\n    if (Array.isArray(source)) {\n      const selected = this.selectSoundFromList(source);\n      if (selected === undefined) {\n        return undefined;\n      }\n\n      return {\n        source: selected.source,\n        volume: selected.volume ?? 1.0,\n      };\n    }\n\n    if (typeof source === 'object' && 'source' in source) {\n      return {\n        source: this.normalizeBuiltinAudio(source.source),\n        volume: source.volume ?? 1.0,\n      };\n    }\n\n    return { source, volume: 1.0 };\n  }\n\n  private normalizeBuiltinAudio(source: AudioSourceType): AudioSourceType {\n    if (isBuiltinAudioClip(source)) {\n      return getBuiltinAudioPath(source);\n    }\n    return source;\n  }\n\n  play(audio: AudioSourceType | AudioConfig | AudioConfig[], loop = false): PlayHandle {\n    const normalized = this.normalizeSoundSource(audio);\n    if (normalized === undefined) {\n      const handle = new PlayHandle();\n      handle._markPlayoutDone();\n      return handle;\n    }\n\n    const { source, volume } = normalized;\n    const playHandle = new PlayHandle();\n\n    const task = Task.from(async ({ signal }) => {\n      await this.playTask({ playHandle, sound: source, volume, loop, signal });\n    });\n\n    task.addDoneCallback(() => {\n      playHandle._markPlayoutDone();\n      this.playTasks.splice(this.playTasks.indexOf(task), 1);\n    });\n\n    this.playTasks.push(task);\n    return playHandle;\n  }\n\n  /**\n   * Start the background audio system, publishing the audio track\n   * and beginning playback of any configured ambient sound.\n   *\n   * If `ambientSound` is provided (and contains file paths), they will loop\n   * automatically. If `ambientSound` contains AsyncIterators, they are assumed\n   * to be already infinite or looped.\n   *\n   * @param options - Options for starting background audio playback\n   */\n  async start(options: BackgroundAudioStartOptions): Promise<void> {\n    const { room, agentSession, trackPublishOptions } = options;\n    this.room = room;\n    this.agentSession = agentSession;\n    this.trackPublishOptions = trackPublishOptions;\n\n    this.closed = false;\n\n    await this.publishTrack();\n\n    // TODO (Brian): check job context is not fake\n\n    this.mixerTask = Task.from(async () => {\n      try {\n        await this.runMixerTask();\n      } catch (err) {\n        if (this.closed) return; // expected when AudioSource is closed\n        throw err;\n      }\n    });\n\n    this.room.on('reconnected', this.onReconnected);\n\n    this.agentSession?.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n    if (!this.ambientSound) return;\n\n    const normalized = this.normalizeSoundSource(this.ambientSound);\n    if (!normalized) return;\n\n    const { source, volume } = normalized;\n    const selectedSound: AudioConfig = { source, volume, probability: 1.0 };\n    this.ambientHandle = this.play(selectedSound, typeof source === 'string');\n  }\n\n  /**\n   * Close and cleanup the background audio system\n   */\n  async close(): Promise<void> {\n    this.closed = true;\n\n    await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);\n\n    if (this.republishTask) {\n      await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);\n    }\n\n    await this.audioMixer.aclose();\n    await this.audioSource.close();\n\n    if (this.mixerTask) {\n      await this.mixerTask.cancelAndWait(TASK_TIMEOUT_MS);\n    }\n\n    this.agentSession?.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n    this.room?.off('reconnected', this.onReconnected);\n\n    if (this.publication && this.publication.sid) {\n      await this.room?.localParticipant?.unpublishTrack(this.publication.sid);\n    }\n  }\n\n  /**\n   * Get the current track publication\n   */\n  getPublication(): LocalTrackPublication | undefined {\n    return this.publication;\n  }\n\n  private async publishTrack(): Promise<void> {\n    if (this.publication !== undefined) {\n      return;\n    }\n\n    const track = LocalAudioTrack.createAudioTrack('background_audio', this.audioSource);\n\n    if (this.room?.localParticipant === undefined) {\n      throw new Error('Local participant not available');\n    }\n\n    const publication = await this.room.localParticipant.publishTrack(\n      track,\n      this.trackPublishOptions ?? new TrackPublishOptions(),\n    );\n\n    this.publication = publication;\n    this.#logger.debug(`Background audio track published: ${this.publication.sid}`);\n  }\n\n  private onReconnected = (): void => {\n    if (this.republishTask) {\n      this.republishTask.cancel();\n    }\n\n    this.publication = undefined;\n    this.republishTask = Task.from(async () => {\n      await this.republishTrackTask();\n    });\n  };\n\n  private async republishTrackTask(): Promise<void> {\n    // TODO (Brian): add lock protection when implementing lock\n    await this.publishTrack();\n  }\n\n  private async runMixerTask(): Promise<void> {\n    for await (const frame of this.audioMixer) {\n      await this.audioSource.captureFrame(frame);\n    }\n  }\n\n  private onAgentStateChanged = (ev: AgentStateChangedEvent): void => {\n    if (!this.thinkingSound) {\n      return;\n    }\n\n    if (ev.newState === 'thinking') {\n      if (this.thinkingHandle && !this.thinkingHandle.done()) {\n        return;\n      }\n\n      const normalized = this.normalizeSoundSource(this.thinkingSound);\n      if (normalized) {\n        const { source, volume } = normalized;\n        const selectedSound: AudioConfig = { source, volume, probability: 1.0 };\n        // Loop thinking sound while in thinking state (same as ambient)\n        this.thinkingHandle = this.play(selectedSound, typeof source === 'string');\n      }\n    } else {\n      this.thinkingHandle?.stop();\n    }\n  };\n\n  // Note: Python uses numpy, TS uses typed arrays for equivalent logic\n  private applyVolumeToFrame(frame: AudioFrame, volume: number): AudioFrame {\n    const int16Data = new Int16Array(\n      frame.data.buffer,\n      frame.data.byteOffset,\n      frame.data.byteLength / 2,\n    );\n    const float32Data = new Float32Array(int16Data.length);\n\n    for (let i = 0; i < int16Data.length; i++) {\n      float32Data[i] = int16Data[i]!;\n    }\n\n    const volumeFactor = 10 ** Math.log10(volume);\n    for (let i = 0; i < float32Data.length; i++) {\n      float32Data[i]! *= volumeFactor;\n    }\n\n    const outputData = new Int16Array(float32Data.length);\n    for (let i = 0; i < float32Data.length; i++) {\n      const clipped = Math.max(-32768, Math.min(32767, float32Data[i]!));\n      outputData[i] = Math.round(clipped);\n    }\n\n    return new AudioFrame(outputData, frame.sampleRate, frame.channels, frame.samplesPerChannel);\n  }\n\n  private async playTask({\n    playHandle,\n    sound,\n    volume,\n    loop,\n    signal,\n  }: {\n    playHandle: PlayHandle;\n    sound: AudioSourceType;\n    volume: number;\n    loop: boolean;\n    signal: AbortSignal;\n  }): Promise<void> {\n    if (isBuiltinAudioClip(sound)) {\n      sound = getBuiltinAudioPath(sound);\n    }\n\n    let audioStream: AsyncIterable<AudioFrame>;\n    if (typeof sound === 'string') {\n      audioStream = loop\n        ? loopAudioFramesFromFile(sound, { abortSignal: signal })\n        : audioFramesFromFile(sound, { abortSignal: signal });\n    } else {\n      audioStream = sound;\n    }\n\n    const applyVolume = this.applyVolumeToFrame.bind(this);\n    async function* genWrapper(): AsyncGenerator<AudioFrame> {\n      for await (const frame of audioStream) {\n        if (signal.aborted || playHandle.done()) break;\n        yield volume !== 1.0 ? applyVolume(frame, volume) : frame;\n      }\n      playHandle._markPlayoutDone();\n    }\n\n    const gen = genWrapper();\n    try {\n      this.audioMixer.addStream(gen);\n      await playHandle.waitForPlayout();\n    } finally {\n      this.audioMixer.removeStream(gen);\n      playHandle._markPlayoutDone();\n\n      if (playHandle.done()) {\n        await gen.return(undefined);\n      }\n    }\n  }\n}\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () =>\n  typeof document === 'undefined'\n    ? new URL(`file:${__filename}`).href\n    : (document.currentScript && document.currentScript.src) ||\n      new URL('main.js', document.baseURI).href\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,OAClD,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEpC,IAAM,gBAAgC,iCAAiB;ADR9D,sBAQO;AACP,uBAA8B;AAC9B,sBAA8B;AAC9B,mBAA6D;AAC7D,iBAAoB;AACpB,mBAA4C;AAE5C,oBAAoE;AAEpE,MAAM,kBAAkB;AAEjB,IAAK,mBAAL,kBAAKA,sBAAL;AACL,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,sBAAmB;AAHT,SAAAA;AAAA,GAAA;AAML,SAAS,mBACd,QAC4B;AAC5B,SACE,OAAO,WAAW,YAClB,OAAO,OAAO,gBAAgB,EAAE,SAAS,MAA0B;AAEvE;AAEO,SAAS,oBAAoB,MAAgC;AAClE,QAAM,oBAAgB,2BAAK,8BAAQ,+BAAc,aAAe,CAAC,GAAG,iBAAiB;AACrF,aAAO,uBAAK,eAAe,IAAI;AACjC;AAuCA,MAAM,yBAAyB;AAExB,MAAM,WAAW;AAAA,EACd,aAAa,IAAI,oBAAa;AAAA,EAC9B,aAAa,IAAI,oBAAa;AAAA,EAEtC,OAAgB;AACd,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,OAAa;AACX,QAAI,KAAK,KAAK,EAAG;AAEjB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,MAAM,iBAAgC;AACpC,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,mBAAyB;AACvB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAAA,EACF;AACF;AAuBO,MAAM,sBAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAA0B,CAAC;AAAA,EAC3B,cAAc,IAAI,4BAAY,MAAO,GAAG,sBAAsB;AAAA,EAC9D;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,SAAS;AAAA;AAAA,EAIjB,cAAU,gBAAI;AAAA,EAEd,YAAY,SAAwC;AAClD,UAAM,EAAE,cAAc,eAAe,kBAAkB,IAAI,IAAI,WAAW,CAAC;AAE3E,SAAK,eAAe;AACpB,SAAK,gBAAgB;AACrB,SAAK,kBAAkB;AAEvB,SAAK,aAAa,IAAI,2BAAW,MAAO,GAAG;AAAA,MACzC,WAAW;AAAA;AAAA,MACX,UAAU;AAAA,MACV,iBAAiB,KAAK;AAAA,IACxB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,oBAAoB,QAAgD;AAC1E,UAAM,mBAAmB,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,MAAM,eAAe,IAAM,CAAC;AAE1F,QAAI,oBAAoB,GAAG;AACzB,aAAO;AAAA,IACT;AAEA,QAAI,mBAAmB,KAAO,KAAK,OAAO,IAAI,kBAAkB;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,oBAAoB,IAAM,IAAM;AACxD,UAAM,IAAI,KAAK,OAAO,IAAI,KAAK,IAAI,kBAAkB,CAAG;AACxD,QAAI,aAAa;AAEjB,eAAW,SAAS,QAAQ;AAC1B,YAAM,OAAO,MAAM,eAAe;AAClC,UAAI,QAAQ,GAAG;AACb;AAAA,MACF;AAEA,YAAM,WAAW,OAAO;AACxB,oBAAc;AAEd,UAAI,KAAK,YAAY;AACnB,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,OAAO,OAAO,SAAS,CAAC;AAAA,EACjC;AAAA,EAEQ,qBACN,QACyD;AACzD,QAAI,WAAW,QAAW;AACxB,aAAO;AAAA,IACT;AAEA,QAAI,OAAO,WAAW,UAAU;AAC9B,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,MAAM;AAAA,QACzC,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,YAAM,WAAW,KAAK,oBAAoB,MAAM;AAChD,UAAI,aAAa,QAAW;AAC1B,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL,QAAQ,SAAS;AAAA,QACjB,QAAQ,SAAS,UAAU;AAAA,MAC7B;AAAA,IACF;AAEA,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,OAAO,MAAM;AAAA,QAChD,QAAQ,OAAO,UAAU;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ,QAAQ,EAAI;AAAA,EAC/B;AAAA,EAEQ,sBAAsB,QAA0C;AACtE,QAAI,mBAAmB,MAAM,GAAG;AAC9B,aAAO,oBAAoB,MAAM;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,OAAsD,OAAO,OAAmB;AACnF,UAAM,aAAa,KAAK,qBAAqB,KAAK;AAClD,QAAI,eAAe,QAAW;AAC5B,YAAM,SAAS,IAAI,WAAW;AAC9B,aAAO,iBAAiB;AACxB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,aAAa,IAAI,WAAW;AAElC,UAAM,OAAO,kBAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AAC3C,YAAM,KAAK,SAAS,EAAE,YAAY,OAAO,QAAQ,QAAQ,MAAM,OAAO,CAAC;AAAA,IACzE,CAAC;AAED,SAAK,gBAAgB,MAAM;AACzB,iBAAW,iBAAiB;AAC5B,WAAK,UAAU,OAAO,KAAK,UAAU,QAAQ,IAAI,GAAG,CAAC;AAAA,IACvD,CAAC;AAED,SAAK,UAAU,KAAK,IAAI;AACxB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,MAAM,SAAqD;AA3RnE;AA4RI,UAAM,EAAE,MAAM,cAAc,oBAAoB,IAAI;AACpD,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB;AAE3B,SAAK,SAAS;AAEd,UAAM,KAAK,aAAa;AAIxB,SAAK,YAAY,kBAAK,KAAK,YAAY;AACrC,UAAI;AACF,cAAM,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK;AACZ,YAAI,KAAK,OAAQ;AACjB,cAAM;AAAA,MACR;AAAA,IACF,CAAC;AAED,SAAK,KAAK,GAAG,eAAe,KAAK,aAAa;AAE9C,eAAK,iBAAL,mBAAmB,GAAG,qCAAuB,mBAAmB,KAAK;AACrE,QAAI,CAAC,KAAK,aAAc;AAExB,UAAM,aAAa,KAAK,qBAAqB,KAAK,YAAY;AAC9D,QAAI,CAAC,WAAY;AAEjB,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,gBAA6B,EAAE,QAAQ,QAAQ,aAAa,EAAI;AACtE,SAAK,gBAAgB,KAAK,KAAK,eAAe,OAAO,WAAW,QAAQ;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAhU/B;AAiUI,SAAK,SAAS;AAEd,cAAM,4BAAc,KAAK,WAAW,eAAe;AAEnD,QAAI,KAAK,eAAe;AACtB,YAAM,KAAK,cAAc,cAAc,eAAe;AAAA,IACxD;AAEA,UAAM,KAAK,WAAW,OAAO;AAC7B,UAAM,KAAK,YAAY,MAAM;AAE7B,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK,UAAU,cAAc,eAAe;AAAA,IACpD;AAEA,eAAK,iBAAL,mBAAmB,IAAI,qCAAuB,mBAAmB,KAAK;AACtE,eAAK,SAAL,mBAAW,IAAI,eAAe,KAAK;AAEnC,QAAI,KAAK,eAAe,KAAK,YAAY,KAAK;AAC5C,cAAM,gBAAK,SAAL,mBAAW,qBAAX,mBAA6B,eAAe,KAAK,YAAY;AAAA,IACrE;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAoD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,eAA8B;AA/V9C;AAgWI,QAAI,KAAK,gBAAgB,QAAW;AAClC;AAAA,IACF;AAEA,UAAM,QAAQ,gCAAgB,iBAAiB,oBAAoB,KAAK,WAAW;AAEnF,UAAI,UAAK,SAAL,mBAAW,sBAAqB,QAAW;AAC7C,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,UAAM,cAAc,MAAM,KAAK,KAAK,iBAAiB;AAAA,MACnD;AAAA,MACA,KAAK,uBAAuB,IAAI,oCAAoB;AAAA,IACtD;AAEA,SAAK,cAAc;AACnB,SAAK,QAAQ,MAAM,qCAAqC,KAAK,YAAY,GAAG,EAAE;AAAA,EAChF;AAAA,EAEQ,gBAAgB,MAAY;AAClC,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,SAAK,cAAc;AACnB,SAAK,gBAAgB,kBAAK,KAAK,YAAY;AACzC,YAAM,KAAK,mBAAmB;AAAA,IAChC,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,qBAAoC;AAEhD,UAAM,KAAK,aAAa;AAAA,EAC1B;AAAA,EAEA,MAAc,eAA8B;AAC1C,qBAAiB,SAAS,KAAK,YAAY;AACzC,YAAM,KAAK,YAAY,aAAa,KAAK;AAAA,IAC3C;AAAA,EACF;AAAA,EAEQ,sBAAsB,CAAC,OAAqC;AAzYtE;AA0YI,QAAI,CAAC,KAAK,eAAe;AACvB;AAAA,IACF;AAEA,QAAI,GAAG,aAAa,YAAY;AAC9B,UAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,KAAK,GAAG;AACtD;AAAA,MACF;AAEA,YAAM,aAAa,KAAK,qBAAqB,KAAK,aAAa;AAC/D,UAAI,YAAY;AACd,cAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,cAAM,gBAA6B,EAAE,QAAQ,QAAQ,aAAa,EAAI;AAEtE,aAAK,iBAAiB,KAAK,KAAK,eAAe,OAAO,WAAW,QAAQ;AAAA,MAC3E;AAAA,IACF,OAAO;AACL,iBAAK,mBAAL,mBAAqB;AAAA,IACvB;AAAA,EACF;AAAA;AAAA,EAGQ,mBAAmB,OAAmB,QAA4B;AACxE,UAAM,YAAY,IAAI;AAAA,MACpB,MAAM,KAAK;AAAA,MACX,MAAM,KAAK;AAAA,MACX,MAAM,KAAK,aAAa;AAAA,IAC1B;AACA,UAAM,cAAc,IAAI,aAAa,UAAU,MAAM;AAErD,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,kBAAY,CAAC,IAAI,UAAU,CAAC;AAAA,IAC9B;AAEA,UAAM,eAAe,MAAM,KAAK,MAAM,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,kBAAY,CAAC,KAAM;AAAA,IACrB;AAEA,UAAM,aAAa,IAAI,WAAW,YAAY,MAAM;AACpD,aAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,YAAM,UAAU,KAAK,IAAI,QAAQ,KAAK,IAAI,OAAO,YAAY,CAAC,CAAE,CAAC;AACjE,iBAAW,CAAC,IAAI,KAAK,MAAM,OAAO;AAAA,IACpC;AAEA,WAAO,IAAI,2BAAW,YAAY,MAAM,YAAY,MAAM,UAAU,MAAM,iBAAiB;AAAA,EAC7F;AAAA,EAEA,MAAc,SAAS;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMkB;AAChB,QAAI,mBAAmB,KAAK,GAAG;AAC7B,cAAQ,oBAAoB,KAAK;AAAA,IACnC;AAEA,QAAI;AACJ,QAAI,OAAO,UAAU,UAAU;AAC7B,oBAAc,WACV,sCAAwB,OAAO,EAAE,aAAa,OAAO,CAAC,QACtD,kCAAoB,OAAO,EAAE,aAAa,OAAO,CAAC;AAAA,IACxD,OAAO;AACL,oBAAc;AAAA,IAChB;AAEA,UAAM,cAAc,KAAK,mBAAmB,KAAK,IAAI;AACrD,oBAAgB,aAAyC;AACvD,uBAAiB,SAAS,aAAa;AACrC,YAAI,OAAO,WAAW,WAAW,KAAK,EAAG;AACzC,cAAM,WAAW,IAAM,YAAY,OAAO,MAAM,IAAI;AAAA,MACtD;AACA,iBAAW,iBAAiB;AAAA,IAC9B;AAEA,UAAM,MAAM,WAAW;AACvB,QAAI;AACF,WAAK,WAAW,UAAU,GAAG;AAC7B,YAAM,WAAW,eAAe;AAAA,IAClC,UAAE;AACA,WAAK,WAAW,aAAa,GAAG;AAChC,iBAAW,iBAAiB;AAE5B,UAAI,WAAW,KAAK,GAAG;AACrB,cAAM,IAAI,OAAO,MAAS;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AACF;","names":["BuiltinAudioClip"]}

package/dist/voice/background_audio.d.cts

@@ -22,7 +22,7 @@ export interface BackgroundAudioPlayerOptions {
     ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];
     /**
      * Sound to play when the agent is thinking.
-     * TODO (Brian): Implement thinking sound when AudioMixer becomes available
+     * Plays when agent state changes to 'thinking' and stops when it changes to other states.
      */
     thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];
     /**
@@ -50,15 +50,16 @@ export declare class PlayHandle {
  * This class handles playing ambient sounds and manages audio track publishing.
  * It supports:
  * - Continuous ambient sound playback with looping
+ * - Thinking sound playback during agent processing
+ * - Multiple simultaneous audio streams via AudioMixer
  * - Volume control and probability-based sound selection
  * - Integration with LiveKit rooms and agent sessions
  *
- * Note: Thinking sound not yet supported
- *
  * @example
  * ```typescript
  * const player = new BackgroundAudioPlayer({
  *   ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },
+ *   thinkingSound: { source: BuiltinAudioClip.KEYBOARD_TYPING, volume: 0.6 },
  * });
  *
  * await player.start({ room, agentSession });
@@ -68,8 +69,11 @@ export declare class BackgroundAudioPlayer {
     #private;
     private ambientSound?;
     private thinkingSound?;
+    private streamTimeoutMs;
     private playTasks;
     private audioSource;
+    private audioMixer;
+    private mixerTask?;
     private room?;
     private agentSession?;
     private publication?;
@@ -77,6 +81,7 @@ export declare class BackgroundAudioPlayer {
     private republishTask?;
     private ambientHandle?;
     private thinkingHandle?;
+    private closed;
     constructor(options?: BackgroundAudioPlayerOptions);
     /**
      * Select a sound from a list of background sound based on probability weights
@@ -108,7 +113,9 @@ export declare class BackgroundAudioPlayer {
     private publishTrack;
     private onReconnected;
     private republishTrackTask;
+    private runMixerTask;
     private onAgentStateChanged;
+    private applyVolumeToFrame;
     private playTask;
 }
 //# sourceMappingURL=background_audio.d.ts.map

package/dist/voice/background_audio.d.ts

@@ -22,7 +22,7 @@ export interface BackgroundAudioPlayerOptions {
     ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];
     /**
      * Sound to play when the agent is thinking.
-     * TODO (Brian): Implement thinking sound when AudioMixer becomes available
+     * Plays when agent state changes to 'thinking' and stops when it changes to other states.
      */
     thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];
     /**
@@ -50,15 +50,16 @@ export declare class PlayHandle {
  * This class handles playing ambient sounds and manages audio track publishing.
  * It supports:
  * - Continuous ambient sound playback with looping
+ * - Thinking sound playback during agent processing
+ * - Multiple simultaneous audio streams via AudioMixer
  * - Volume control and probability-based sound selection
  * - Integration with LiveKit rooms and agent sessions
  *
- * Note: Thinking sound not yet supported
- *
  * @example
  * ```typescript
  * const player = new BackgroundAudioPlayer({
  *   ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },
+ *   thinkingSound: { source: BuiltinAudioClip.KEYBOARD_TYPING, volume: 0.6 },
  * });
  *
  * await player.start({ room, agentSession });
@@ -68,8 +69,11 @@ export declare class BackgroundAudioPlayer {
     #private;
     private ambientSound?;
     private thinkingSound?;
+    private streamTimeoutMs;
     private playTasks;
     private audioSource;
+    private audioMixer;
+    private mixerTask?;
     private room?;
     private agentSession?;
     private publication?;
@@ -77,6 +81,7 @@ export declare class BackgroundAudioPlayer {
     private republishTask?;
     private ambientHandle?;
     private thinkingHandle?;
+    private closed;
     constructor(options?: BackgroundAudioPlayerOptions);
     /**
      * Select a sound from a list of background sound based on probability weights
@@ -108,7 +113,9 @@ export declare class BackgroundAudioPlayer {
     private publishTrack;
     private onReconnected;
     private republishTrackTask;
+    private runMixerTask;
     private onAgentStateChanged;
+    private applyVolumeToFrame;
     private playTask;
 }
 //# sourceMappingURL=background_audio.d.ts.map

package/dist/voice/background_audio.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"background_audio.d.ts","sourceRoot":"","sources":["../../src/voice/background_audio.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAGV,KAAK,qBAAqB,EAC1B,KAAK,IAAI,EACT,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAM3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAKvD,oBAAY,gBAAgB;IAC1B,eAAe,wBAAwB;IACvC,eAAe,wBAAwB;IACvC,gBAAgB,yBAAyB;CAC1C;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,GACpD,MAAM,IAAI,gBAAgB,CAK5B;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAGlE;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,gBAAgB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAEpF,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,YAAY,CAAC,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;IAE7D;;;OAGG;IACH,aAAa,CAAC,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;IAE9D;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAMD,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAsB;IAExC,IAAI,IAAI,OAAO;IAIf,IAAI,IAAI,IAAI;IAUN,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrC,gBAAgB,IAAI,IAAI;CAKzB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,qBAAqB;;IAChC,OAAO,CAAC,YAAY,CAAC,CAAgD;IACrE,OAAO,CAAC,aAAa,CAAC,CAAgD;IAEtE,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,WAAW,CAAqD;IAExE,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,WAAW,CAAC,CAAwB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAsB;IAClD,OAAO,CAAC,aAAa,CAAC,CAAa;IAEnC,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,cAAc,CAAC,CAAa;gBAMxB,OAAO,CAAC,EAAE,4BAA4B;IAYlD;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAgC3B,OAAO,CAAC,oBAAoB;IAoC5B,OAAO,CAAC,qBAAqB;IAO7B,IAAI,CAAC,KAAK,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,EAAE,IAAI,UAAQ,GAAG,UAAU;IAwBpF;;;;;;;;;OASG;IACG,KAAK,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBhE;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB5B;;OAEG;IACH,cAAc,IAAI,qBAAqB,GAAG,SAAS;YAIrC,YAAY;IAoB1B,OAAO,CAAC,aAAa,CASnB;YAEY,kBAAkB;IAKhC,OAAO,CAAC,mBAAmB,CAczB;YAEY,QAAQ;CAsEvB"}
+{"version":3,"file":"background_audio.d.ts","sourceRoot":"","sources":["../../src/voice/background_audio.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAIV,KAAK,qBAAqB,EAC1B,KAAK,IAAI,EACT,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAM3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAKvD,oBAAY,gBAAgB;IAC1B,eAAe,wBAAwB;IACvC,eAAe,wBAAwB;IACvC,gBAAgB,yBAAyB;CAC1C;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,GACpD,MAAM,IAAI,gBAAgB,CAK5B;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAGlE;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,gBAAgB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAEpF,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,YAAY,CAAC,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;IAE7D;;;OAGG;IACH,aAAa,CAAC,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;IAE9D;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAMD,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAsB;IAExC,IAAI,IAAI,OAAO;IAIf,IAAI,IAAI,IAAI;IAUN,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrC,gBAAgB,IAAI,IAAI;CAKzB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,qBAAqB;;IAChC,OAAO,CAAC,YAAY,CAAC,CAAgD;IACrE,OAAO,CAAC,aAAa,CAAC,CAAgD;IACtE,OAAO,CAAC,eAAe,CAAS;IAEhC,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,WAAW,CAAqD;IACxE,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,SAAS,CAAC,CAAa;IAE/B,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,WAAW,CAAC,CAAwB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAsB;IAClD,OAAO,CAAC,aAAa,CAAC,CAAa;IAEnC,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,cAAc,CAAC,CAAa;IAEpC,OAAO,CAAC,MAAM,CAAQ;gBAMV,OAAO,CAAC,EAAE,4BAA4B;IAclD;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAgC3B,OAAO,CAAC,oBAAoB;IAoC5B,OAAO,CAAC,qBAAqB;IAO7B,IAAI,CAAC,KAAK,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,EAAE,IAAI,UAAQ,GAAG,UAAU;IAwBpF;;;;;;;;;OASG;IACG,KAAK,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,IAAI,CAAC;IAkChE;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAwB5B;;OAEG;IACH,cAAc,IAAI,qBAAqB,GAAG,SAAS;YAIrC,YAAY;IAoB1B,OAAO,CAAC,aAAa,CASnB;YAEY,kBAAkB;YAKlB,YAAY;IAM1B,OAAO,CAAC,mBAAmB,CAoBzB;IAGF,OAAO,CAAC,kBAAkB;YA0BZ,QAAQ;CAgDvB"}

package/dist/voice/background_audio.js

@@ -1,5 +1,6 @@
 import {
   AudioFrame,
+  AudioMixer,
   AudioSource,
   LocalAudioTrack,
   TrackPublishOptions
@@ -50,8 +51,11 @@ class PlayHandle {
 class BackgroundAudioPlayer {
   ambientSound;
   thinkingSound;
+  streamTimeoutMs;
   playTasks = [];
   audioSource = new AudioSource(48e3, 1, AUDIO_SOURCE_BUFFER_MS);
+  audioMixer;
+  mixerTask;
   room;
   agentSession;
   publication;
@@ -59,15 +63,20 @@ class BackgroundAudioPlayer {
   republishTask;
   ambientHandle;
   thinkingHandle;
+  closed = true;
   // TODO (Brian): add lock
   #logger = log();
   constructor(options) {
-    const { ambientSound, thinkingSound } = options || {};
+    const { ambientSound, thinkingSound, streamTimeoutMs = 200 } = options || {};
     this.ambientSound = ambientSound;
     this.thinkingSound = thinkingSound;
-    if (this.thinkingSound) {
-      this.#logger.warn("thinkingSound is not yet supported");
-    }
+    this.streamTimeoutMs = streamTimeoutMs;
+    this.audioMixer = new AudioMixer(48e3, 1, {
+      blocksize: 4800,
+      // 100ms at 48kHz
+      capacity: 1,
+      streamTimeoutMs: this.streamTimeoutMs
+    });
   }
   /**
    * Select a sound from a list of background sound based on probability weights
@@ -166,7 +175,16 @@ class BackgroundAudioPlayer {
     this.room = room;
     this.agentSession = agentSession;
     this.trackPublishOptions = trackPublishOptions;
+    this.closed = false;
     await this.publishTrack();
+    this.mixerTask = Task.from(async () => {
+      try {
+        await this.runMixerTask();
+      } catch (err) {
+        if (this.closed) return;
+        throw err;
+      }
+    });
     this.room.on("reconnected", this.onReconnected);
     (_a = this.agentSession) == null ? void 0 : _a.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
     if (!this.ambientSound) return;
@@ -181,11 +199,16 @@ class BackgroundAudioPlayer {
    */
   async close() {
     var _a, _b, _c, _d;
+    this.closed = true;
     await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);
     if (this.republishTask) {
       await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);
     }
+    await this.audioMixer.aclose();
     await this.audioSource.close();
+    if (this.mixerTask) {
+      await this.mixerTask.cancelAndWait(TASK_TIMEOUT_MS);
+    }
     (_a = this.agentSession) == null ? void 0 : _a.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
     (_b = this.room) == null ? void 0 : _b.off("reconnected", this.onReconnected);
     if (this.publication && this.publication.sid) {
@@ -226,6 +249,11 @@ class BackgroundAudioPlayer {
   async republishTrackTask() {
     await this.publishTrack();
   }
+  async runMixerTask() {
+    for await (const frame of this.audioMixer) {
+      await this.audioSource.captureFrame(frame);
+    }
+  }
   onAgentStateChanged = (ev) => {
     var _a;
     if (!this.thinkingSound) {
@@ -235,10 +263,38 @@ class BackgroundAudioPlayer {
       if (this.thinkingHandle && !this.thinkingHandle.done()) {
         return;
       }
+      const normalized = this.normalizeSoundSource(this.thinkingSound);
+      if (normalized) {
+        const { source, volume } = normalized;
+        const selectedSound = { source, volume, probability: 1 };
+        this.thinkingHandle = this.play(selectedSound, typeof source === "string");
+      }
     } else {
       (_a = this.thinkingHandle) == null ? void 0 : _a.stop();
     }
   };
+  // Note: Python uses numpy, TS uses typed arrays for equivalent logic
+  applyVolumeToFrame(frame, volume) {
+    const int16Data = new Int16Array(
+      frame.data.buffer,
+      frame.data.byteOffset,
+      frame.data.byteLength / 2
+    );
+    const float32Data = new Float32Array(int16Data.length);
+    for (let i = 0; i < int16Data.length; i++) {
+      float32Data[i] = int16Data[i];
+    }
+    const volumeFactor = 10 ** Math.log10(volume);
+    for (let i = 0; i < float32Data.length; i++) {
+      float32Data[i] *= volumeFactor;
+    }
+    const outputData = new Int16Array(float32Data.length);
+    for (let i = 0; i < float32Data.length; i++) {
+      const clipped = Math.max(-32768, Math.min(32767, float32Data[i]));
+      outputData[i] = Math.round(clipped);
+    }
+    return new AudioFrame(outputData, frame.sampleRate, frame.channels, frame.samplesPerChannel);
+  }
   async playTask({
     playHandle,
     sound,
@@ -249,45 +305,30 @@ class BackgroundAudioPlayer {
     if (isBuiltinAudioClip(sound)) {
       sound = getBuiltinAudioPath(sound);
     }
+    let audioStream;
     if (typeof sound === "string") {
-      sound = loop ? loopAudioFramesFromFile(sound, { abortSignal: signal }) : audioFramesFromFile(sound, { abortSignal: signal });
+      audioStream = loop ? loopAudioFramesFromFile(sound, { abortSignal: signal }) : audioFramesFromFile(sound, { abortSignal: signal });
+    } else {
+      audioStream = sound;
     }
-    try {
-      for await (const frame of sound) {
+    const applyVolume = this.applyVolumeToFrame.bind(this);
+    async function* genWrapper() {
+      for await (const frame of audioStream) {
         if (signal.aborted || playHandle.done()) break;
-        let processedFrame;
-        if (volume !== 1) {
-          const int16Data = new Int16Array(
-            frame.data.buffer,
-            frame.data.byteOffset,
-            frame.data.byteLength / 2
-          );
-          const float32Data = new Float32Array(int16Data.length);
-          for (let i = 0; i < int16Data.length; i++) {
-            float32Data[i] = int16Data[i];
-          }
-          const volumeFactor = 10 ** Math.log10(volume);
-          for (let i = 0; i < float32Data.length; i++) {
-            float32Data[i] *= volumeFactor;
-          }
-          const outputData = new Int16Array(float32Data.length);
-          for (let i = 0; i < float32Data.length; i++) {
-            const clipped = Math.max(-32768, Math.min(32767, float32Data[i]));
-            outputData[i] = Math.round(clipped);
-          }
-          processedFrame = new AudioFrame(
-            outputData,
-            frame.sampleRate,
-            frame.channels,
-            frame.samplesPerChannel
-          );
-        } else {
-          processedFrame = frame;
-        }
-        await this.audioSource.captureFrame(processedFrame);
+        yield volume !== 1 ? applyVolume(frame, volume) : frame;
       }
+      playHandle._markPlayoutDone();
+    }
+    const gen = genWrapper();
+    try {
+      this.audioMixer.addStream(gen);
+      await playHandle.waitForPlayout();
     } finally {
+      this.audioMixer.removeStream(gen);
       playHandle._markPlayoutDone();
+      if (playHandle.done()) {
+        await gen.return(void 0);
+      }
     }
   }
 }