@livekit/agents 1.0.27 → 1.0.31

package/src/voice/background_audio.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import {
   AudioFrame,
+  AudioMixer,
   AudioSource,
   LocalAudioTrack,
   type LocalTrackPublication,
@@ -57,7 +58,7 @@ export interface BackgroundAudioPlayerOptions {
 
   /**
    * Sound to play when the agent is thinking.
-   * TODO (Brian): Implement thinking sound when AudioMixer becomes available
+   * Plays when agent state changes to 'thinking' and stops when it changes to other states.
    */
   thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];
 
@@ -113,15 +114,16 @@ export class PlayHandle {
  * This class handles playing ambient sounds and manages audio track publishing.
  * It supports:
  * - Continuous ambient sound playback with looping
+ * - Thinking sound playback during agent processing
+ * - Multiple simultaneous audio streams via AudioMixer
  * - Volume control and probability-based sound selection
  * - Integration with LiveKit rooms and agent sessions
  *
- * Note: Thinking sound not yet supported
- *
  * @example
  * ```typescript
  * const player = new BackgroundAudioPlayer({
  *   ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },
+ *   thinkingSound: { source: BuiltinAudioClip.KEYBOARD_TYPING, volume: 0.6 },
  * });
  *
  * await player.start({ room, agentSession });
@@ -130,9 +132,12 @@ export class PlayHandle {
 export class BackgroundAudioPlayer {
   private ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];
   private thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];
+  private streamTimeoutMs: number;
 
   private playTasks: Task<void>[] = [];
   private audioSource = new AudioSource(48000, 1, AUDIO_SOURCE_BUFFER_MS);
+  private audioMixer: AudioMixer;
+  private mixerTask?: Task<void>;
 
   private room?: Room;
   private agentSession?: AgentSession;
@@ -143,20 +148,24 @@ export class BackgroundAudioPlayer {
   private ambientHandle?: PlayHandle;
   private thinkingHandle?: PlayHandle;
 
+  private closed = true;
+
   // TODO (Brian): add lock
 
   #logger = log();
 
   constructor(options?: BackgroundAudioPlayerOptions) {
-    const { ambientSound, thinkingSound } = options || {};
+    const { ambientSound, thinkingSound, streamTimeoutMs = 200 } = options || {};
 
     this.ambientSound = ambientSound;
     this.thinkingSound = thinkingSound;
+    this.streamTimeoutMs = streamTimeoutMs;
 
-    if (this.thinkingSound) {
-      this.#logger.warn('thinkingSound is not yet supported');
-      // TODO: Implement thinking sound when AudioMixer becomes available
-    }
+    this.audioMixer = new AudioMixer(48000, 1, {
+      blocksize: 4800, // 100ms at 48kHz
+      capacity: 1,
+      streamTimeoutMs: this.streamTimeoutMs,
+    });
   }
 
   /**
@@ -278,15 +287,24 @@ export class BackgroundAudioPlayer {
     this.agentSession = agentSession;
     this.trackPublishOptions = trackPublishOptions;
 
+    this.closed = false;
+
     await this.publishTrack();
 
     // TODO (Brian): check job context is not fake
 
-    // TODO (Brian): start audio mixer task
+    this.mixerTask = Task.from(async () => {
+      try {
+        await this.runMixerTask();
+      } catch (err) {
+        if (this.closed) return; // expected when AudioSource is closed
+        throw err;
+      }
+    });
+
     this.room.on('reconnected', this.onReconnected);
 
     this.agentSession?.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
-
     if (!this.ambientSound) return;
 
     const normalized = this.normalizeSoundSource(this.ambientSound);
@@ -301,16 +319,21 @@ export class BackgroundAudioPlayer {
    * Close and cleanup the background audio system
    */
   async close(): Promise<void> {
+    this.closed = true;
+
     await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);
 
     if (this.republishTask) {
       await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);
     }
 
-    // TODO (Brian): cancel audio mixer task and close audio mixer
-
+    await this.audioMixer.aclose();
     await this.audioSource.close();
 
+    if (this.mixerTask) {
+      await this.mixerTask.cancelAndWait(TASK_TIMEOUT_MS);
+    }
+
     this.agentSession?.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
     this.room?.off('reconnected', this.onReconnected);
 
@@ -362,6 +385,12 @@ export class BackgroundAudioPlayer {
     await this.publishTrack();
   }
 
+  private async runMixerTask(): Promise<void> {
+    for await (const frame of this.audioMixer) {
+      await this.audioSource.captureFrame(frame);
+    }
+  }
+
   private onAgentStateChanged = (ev: AgentStateChangedEvent): void => {
     if (!this.thinkingSound) {
       return;
@@ -372,12 +401,45 @@ export class BackgroundAudioPlayer {
         return;
       }
 
-      // TODO (Brian): play thinking sound and assign to thinkingHandle
+      const normalized = this.normalizeSoundSource(this.thinkingSound);
+      if (normalized) {
+        const { source, volume } = normalized;
+        const selectedSound: AudioConfig = { source, volume, probability: 1.0 };
+        // Loop thinking sound while in thinking state (same as ambient)
+        this.thinkingHandle = this.play(selectedSound, typeof source === 'string');
+      }
     } else {
       this.thinkingHandle?.stop();
     }
   };
 
+  // Note: Python uses numpy, TS uses typed arrays for equivalent logic
+  private applyVolumeToFrame(frame: AudioFrame, volume: number): AudioFrame {
+    const int16Data = new Int16Array(
+      frame.data.buffer,
+      frame.data.byteOffset,
+      frame.data.byteLength / 2,
+    );
+    const float32Data = new Float32Array(int16Data.length);
+
+    for (let i = 0; i < int16Data.length; i++) {
+      float32Data[i] = int16Data[i]!;
+    }
+
+    const volumeFactor = 10 ** Math.log10(volume);
+    for (let i = 0; i < float32Data.length; i++) {
+      float32Data[i]! *= volumeFactor;
+    }
+
+    const outputData = new Int16Array(float32Data.length);
+    for (let i = 0; i < float32Data.length; i++) {
+      const clipped = Math.max(-32768, Math.min(32767, float32Data[i]!));
+      outputData[i] = Math.round(clipped);
+    }
+
+    return new AudioFrame(outputData, frame.sampleRate, frame.channels, frame.samplesPerChannel);
+  }
+
   private async playTask({
     playHandle,
     sound,
@@ -395,57 +457,35 @@ export class BackgroundAudioPlayer {
       sound = getBuiltinAudioPath(sound);
     }
 
+    let audioStream: AsyncIterable<AudioFrame>;
     if (typeof sound === 'string') {
-      sound = loop
+      audioStream = loop
         ? loopAudioFramesFromFile(sound, { abortSignal: signal })
         : audioFramesFromFile(sound, { abortSignal: signal });
+    } else {
+      audioStream = sound;
     }
 
-    try {
-      for await (const frame of sound) {
+    const applyVolume = this.applyVolumeToFrame.bind(this);
+    async function* genWrapper(): AsyncGenerator<AudioFrame> {
+      for await (const frame of audioStream) {
         if (signal.aborted || playHandle.done()) break;
-
-        let processedFrame: AudioFrame;
-
-        if (volume !== 1.0) {
-          const int16Data = new Int16Array(
-            frame.data.buffer,
-            frame.data.byteOffset,
-            frame.data.byteLength / 2,
-          );
-          const float32Data = new Float32Array(int16Data.length);
-
-          for (let i = 0; i < int16Data.length; i++) {
-            float32Data[i] = int16Data[i]!;
-          }
-
-          const volumeFactor = 10 ** Math.log10(volume);
-          for (let i = 0; i < float32Data.length; i++) {
-            float32Data[i]! *= volumeFactor;
-          }
-
-          const outputData = new Int16Array(float32Data.length);
-          for (let i = 0; i < float32Data.length; i++) {
-            const clipped = Math.max(-32768, Math.min(32767, float32Data[i]!));
-            outputData[i] = Math.round(clipped);
-          }
-
-          processedFrame = new AudioFrame(
-            outputData,
-            frame.sampleRate,
-            frame.channels,
-            frame.samplesPerChannel,
-          );
-        } else {
-          processedFrame = frame;
-        }
-
-        // TODO (Brian): use AudioMixer to add/remove frame streams
-        await this.audioSource.captureFrame(processedFrame);
+        yield volume !== 1.0 ? applyVolume(frame, volume) : frame;
       }
+      playHandle._markPlayoutDone();
+    }
+
+    const gen = genWrapper();
+    try {
+      this.audioMixer.addStream(gen);
+      await playHandle.waitForPlayout();
     } finally {
-      // TODO: the waitForPlayout() may be innaccurate by 400ms
+      this.audioMixer.removeStream(gen);
       playHandle._markPlayoutDone();
+
+      if (playHandle.done()) {
+        await gen.return(undefined);
+      }
     }
   }
 }

package/src/voice/index.ts

@@ -6,6 +6,7 @@ export { AgentSession, type AgentSessionOptions } from './agent_session.js';
 export * from './avatar/index.js';
 export * from './background_audio.js';
 export * from './events.js';
+export { type TimedString } from './io.js';
 export * from './report.js';
 export * from './room_io/index.js';
 export { RunContext } from './run_context.js';

package/src/voice/io.ts

@@ -29,6 +29,20 @@ export type TTSNode = (
   modelSettings: ModelSettings,
 ) => Promise<ReadableStream<AudioFrame> | null>;
 
+/**
+ * A string with timing information for word-level alignment.
+ */
+export interface TimedString {
+  text: string;
+  startTime?: number; // seconds
+  endTime?: number; // seconds
+}
+
+export interface AudioOutputCapabilities {
+  /** Whether this output supports pause/resume functionality */
+  pause: boolean;
+}
+
 export abstract class AudioInput {
   protected deferredStream: DeferredReadableStream<AudioFrame> =
     new DeferredReadableStream<AudioFrame>();
@@ -54,12 +68,15 @@ export abstract class AudioOutput extends EventEmitter {
     interrupted: false,
   };
   protected logger = log();
+  protected readonly capabilities: AudioOutputCapabilities;
 
   constructor(
     public sampleRate?: number,
     protected readonly nextInChain?: AudioOutput,
+    capabilities: AudioOutputCapabilities = { pause: false },
   ) {
     super();
+    this.capabilities = capabilities;
     if (this.nextInChain) {
       this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) =>
         this.onPlaybackFinished(ev),
@@ -67,6 +84,13 @@ export abstract class AudioOutput extends EventEmitter {
     }
   }
 
+  /**
+   * Whether this output and all outputs in the chain support pause/resume.
+   */
+  get canPause(): boolean {
+    return this.capabilities.pause && (this.nextInChain?.canPause ?? true);
+  }
+
   /**
    * Capture an audio frame for playback, frames can be pushed faster than real-time
    */

package/src/voice/recorder_io/recorder_io.ts

@@ -465,7 +465,7 @@ class RecorderAudioOutput extends AudioOutput {
     audioOutput: AudioOutput,
     writeFn: (buf: AudioFrame[]) => void,
   ) {
-    super(audioOutput.sampleRate, audioOutput);
+    super(audioOutput.sampleRate, audioOutput, { pause: true });
     this.recorderIO = recorderIO;
     this.writeFn = writeFn;
   }

package/src/voice/room_io/_output.ts

@@ -328,7 +328,7 @@ export class ParticipantAudioOutput extends AudioOutput {
   private interruptedFuture: Future<void> = new Future();
 
   constructor(room: Room, options: AudioOutputOptions) {
-    super(options.sampleRate);
+    super(options.sampleRate, undefined, { pause: true });
     this.room = room;
     this.options = options;
     this.audioSource = new AudioSource(options.sampleRate, options.numChannels);

package/src/voice/transcription/synchronizer.ts

@@ -362,7 +362,7 @@ class SyncedAudioOutput extends AudioOutput {
     public synchronizer: TranscriptionSynchronizer,
     private nextInChainAudio: AudioOutput,
   ) {
-    super(nextInChainAudio.sampleRate, nextInChainAudio);
+    super(nextInChainAudio.sampleRate, nextInChainAudio, { pause: true });
   }
 
   async captureFrame(frame: AudioFrame): Promise<void> {

package/src/worker.ts

@@ -384,7 +384,7 @@ export class AgentServer {
         try {
           await new Promise((resolve, reject) => {
             this.#session!.on('open', resolve);
-            this.#session!.on('error', (error) => reject(error.message));
+            this.#session!.on('error', (error) => reject(error));
             this.#session!.on('close', (code) => reject(`WebSocket returned ${code}`));
           });
 
@@ -392,14 +392,10 @@ export class AgentServer {
           this.#logger.debug('connected to LiveKit server');
           await this.#runWS(this.#session);
         } catch (e: unknown) {
-          if (e instanceof Error || e instanceof ErrorEvent) {
-            e = e.message;
-          }
-
           if (this.#closed) return;
           if (retries >= this.#opts.maxRetry) {
             throw new WorkerError(
-              `failed to connect to LiveKit server after ${retries} attempts: ${e}`,
+              `failed to connect to LiveKit server (${this.#opts.wsURL}) after ${retries} attempts: ${e}`,
             );
           }
 
@@ -407,7 +403,8 @@ export class AgentServer {
           const delay = Math.min(retries * 2, 10);
 
           this.#logger.warn(
-            `failed to connect to LiveKit server, retrying in ${delay} seconds: ${e} (${retries}/${this.#opts.maxRetry})`,
+            e,
+            `failed to connect to LiveKit server (${this.#opts.wsURL}), retrying in ${delay} seconds: (${retries}/${this.#opts.maxRetry})`,
           );
 
           await new Promise((resolve) => setTimeout(resolve, delay * 1000));