@livekit/agents 1.1.0 → 1.2.0

package/dist/voice/background_audio.js

@@ -0,0 +1,343 @@
+import {
+  AudioFrame,
+  AudioMixer,
+  AudioSource,
+  LocalAudioTrack,
+  TrackPublishOptions
+} from "@livekit/rtc-node";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { audioFramesFromFile, loopAudioFramesFromFile } from "../audio.js";
+import { log } from "../log.js";
+import { Future, Task, cancelAndWait } from "../utils.js";
+import { AgentSessionEventTypes } from "./events.js";
+const TASK_TIMEOUT_MS = 500;
+var BuiltinAudioClip = /* @__PURE__ */ ((BuiltinAudioClip2) => {
+  BuiltinAudioClip2["OFFICE_AMBIENCE"] = "office-ambience.ogg";
+  BuiltinAudioClip2["KEYBOARD_TYPING"] = "keyboard-typing.ogg";
+  BuiltinAudioClip2["KEYBOARD_TYPING2"] = "keyboard-typing2.ogg";
+  return BuiltinAudioClip2;
+})(BuiltinAudioClip || {});
+function isBuiltinAudioClip(source) {
+  return typeof source === "string" && Object.values(BuiltinAudioClip).includes(source);
+}
+function getBuiltinAudioPath(clip) {
+  const resourcesPath = join(dirname(fileURLToPath(import.meta.url)), "../../resources");
+  return join(resourcesPath, clip);
+}
+const AUDIO_SOURCE_BUFFER_MS = 400;
+const STREAM_TIMEOUT_MS = 2e3;
+class PlayHandle {
+  doneFuture = new Future();
+  stopFuture = new Future();
+  done() {
+    return this.doneFuture.done;
+  }
+  stop() {
+    if (this.done()) return;
+    if (!this.stopFuture.done) {
+      this.stopFuture.resolve();
+    }
+    this._markPlayoutDone();
+  }
+  async waitForPlayout() {
+    return this.doneFuture.await;
+  }
+  _markPlayoutDone() {
+    if (!this.doneFuture.done) {
+      this.doneFuture.resolve();
+    }
+  }
+}
+class BackgroundAudioPlayer {
+  ambientSound;
+  thinkingSound;
+  streamTimeoutMs;
+  playTasks = [];
+  audioSource = new AudioSource(48e3, 1, AUDIO_SOURCE_BUFFER_MS);
+  audioMixer;
+  mixerTask;
+  room;
+  agentSession;
+  publication;
+  trackPublishOptions;
+  republishTask;
+  ambientHandle;
+  thinkingHandle;
+  closed = true;
+  // TODO (Brian): add lock
+  #logger = log();
+  constructor(options) {
+    const { ambientSound, thinkingSound, streamTimeoutMs = STREAM_TIMEOUT_MS } = options || {};
+    this.ambientSound = ambientSound;
+    this.thinkingSound = thinkingSound;
+    this.streamTimeoutMs = streamTimeoutMs;
+    this.audioMixer = new AudioMixer(48e3, 1, {
+      blocksize: 4800,
+      // 100ms at 48kHz
+      capacity: 1,
+      streamTimeoutMs: this.streamTimeoutMs
+    });
+  }
+  /**
+   * Select a sound from a list of background sound based on probability weights
+   * Return undefined if no sound is selected (when sum of probabilities < 1.0).
+   */
+  selectSoundFromList(sounds) {
+    const totalProbability = sounds.reduce((sum, sound) => sum + (sound.probability ?? 1), 0);
+    if (totalProbability <= 0) {
+      return void 0;
+    }
+    if (totalProbability < 1 && Math.random() > totalProbability) {
+      return void 0;
+    }
+    const normalizeFactor = totalProbability <= 1 ? 1 : totalProbability;
+    const r = Math.random() * Math.min(totalProbability, 1);
+    let cumulative = 0;
+    for (const sound of sounds) {
+      const prob = sound.probability ?? 1;
+      if (prob <= 0) {
+        continue;
+      }
+      const normProb = prob / normalizeFactor;
+      cumulative += normProb;
+      if (r <= cumulative) {
+        return sound;
+      }
+    }
+    return sounds[sounds.length - 1];
+  }
+  normalizeSoundSource(source) {
+    if (source === void 0) {
+      return void 0;
+    }
+    if (typeof source === "string") {
+      return {
+        source: this.normalizeBuiltinAudio(source),
+        volume: 1
+      };
+    }
+    if (Array.isArray(source)) {
+      const selected = this.selectSoundFromList(source);
+      if (selected === void 0) {
+        return void 0;
+      }
+      return {
+        source: selected.source,
+        volume: selected.volume ?? 1
+      };
+    }
+    if (typeof source === "object" && "source" in source) {
+      return {
+        source: this.normalizeBuiltinAudio(source.source),
+        volume: source.volume ?? 1
+      };
+    }
+    return { source, volume: 1 };
+  }
+  normalizeBuiltinAudio(source) {
+    if (isBuiltinAudioClip(source)) {
+      return getBuiltinAudioPath(source);
+    }
+    return source;
+  }
+  play(audio, loop = false) {
+    const normalized = this.normalizeSoundSource(audio);
+    if (normalized === void 0) {
+      const handle = new PlayHandle();
+      handle._markPlayoutDone();
+      return handle;
+    }
+    const { source, volume } = normalized;
+    const playHandle = new PlayHandle();
+    const task = Task.from(async ({ signal }) => {
+      await this.playTask({ playHandle, sound: source, volume, loop, signal });
+    });
+    task.addDoneCallback(() => {
+      playHandle._markPlayoutDone();
+      this.playTasks.splice(this.playTasks.indexOf(task), 1);
+    });
+    this.playTasks.push(task);
+    return playHandle;
+  }
+  /**
+   * Start the background audio system, publishing the audio track
+   * and beginning playback of any configured ambient sound.
+   *
+   * If `ambientSound` is provided (and contains file paths), they will loop
+   * automatically. If `ambientSound` contains AsyncIterators, they are assumed
+   * to be already infinite or looped.
+   *
+   * @param options - Options for starting background audio playback
+   */
+  async start(options) {
+    var _a;
+    const { room, agentSession, trackPublishOptions } = options;
+    this.room = room;
+    this.agentSession = agentSession;
+    this.trackPublishOptions = trackPublishOptions;
+    this.closed = false;
+    await this.publishTrack();
+    this.mixerTask = Task.from(async () => {
+      try {
+        await this.runMixerTask();
+      } catch (err) {
+        if (this.closed) return;
+        throw err;
+      }
+    });
+    this.room.on("reconnected", this.onReconnected);
+    (_a = this.agentSession) == null ? void 0 : _a.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
+    if (!this.ambientSound) return;
+    const normalized = this.normalizeSoundSource(this.ambientSound);
+    if (!normalized) return;
+    const { source, volume } = normalized;
+    const selectedSound = { source, volume, probability: 1 };
+    this.ambientHandle = this.play(selectedSound, typeof source === "string");
+  }
+  /**
+   * Close and cleanup the background audio system
+   */
+  async close() {
+    var _a, _b, _c, _d;
+    this.closed = true;
+    await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);
+    if (this.republishTask) {
+      await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);
+    }
+    await this.audioMixer.aclose();
+    await this.audioSource.close();
+    if (this.mixerTask) {
+      await this.mixerTask.cancelAndWait(TASK_TIMEOUT_MS);
+    }
+    (_a = this.agentSession) == null ? void 0 : _a.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
+    (_b = this.room) == null ? void 0 : _b.off("reconnected", this.onReconnected);
+    if (this.publication && this.publication.sid) {
+      await ((_d = (_c = this.room) == null ? void 0 : _c.localParticipant) == null ? void 0 : _d.unpublishTrack(this.publication.sid));
+    }
+  }
+  /**
+   * Get the current track publication
+   */
+  getPublication() {
+    return this.publication;
+  }
+  async publishTrack() {
+    var _a;
+    if (this.publication !== void 0) {
+      return;
+    }
+    const track = LocalAudioTrack.createAudioTrack("background_audio", this.audioSource);
+    if (((_a = this.room) == null ? void 0 : _a.localParticipant) === void 0) {
+      throw new Error("Local participant not available");
+    }
+    const publication = await this.room.localParticipant.publishTrack(
+      track,
+      this.trackPublishOptions ?? new TrackPublishOptions()
+    );
+    this.publication = publication;
+    this.#logger.debug(`Background audio track published: ${this.publication.sid}`);
+  }
+  onReconnected = () => {
+    if (this.republishTask) {
+      this.republishTask.cancel();
+    }
+    this.publication = void 0;
+    this.republishTask = Task.from(async () => {
+      await this.republishTrackTask();
+    });
+  };
+  async republishTrackTask() {
+    await this.publishTrack();
+  }
+  async runMixerTask() {
+    for await (const frame of this.audioMixer) {
+      await this.audioSource.captureFrame(frame);
+    }
+  }
+  onAgentStateChanged = (ev) => {
+    var _a;
+    if (!this.thinkingSound) {
+      return;
+    }
+    if (ev.newState === "thinking") {
+      if (this.thinkingHandle && !this.thinkingHandle.done()) {
+        return;
+      }
+      const normalized = this.normalizeSoundSource(this.thinkingSound);
+      if (normalized) {
+        const { source, volume } = normalized;
+        const selectedSound = { source, volume, probability: 1 };
+        this.thinkingHandle = this.play(selectedSound, typeof source === "string");
+      }
+    } else {
+      (_a = this.thinkingHandle) == null ? void 0 : _a.stop();
+    }
+  };
+  // Note: Python uses numpy, TS uses typed arrays for equivalent logic
+  applyVolumeToFrame(frame, volume) {
+    const int16Data = new Int16Array(
+      frame.data.buffer,
+      frame.data.byteOffset,
+      frame.data.byteLength / 2
+    );
+    const float32Data = new Float32Array(int16Data.length);
+    for (let i = 0; i < int16Data.length; i++) {
+      float32Data[i] = int16Data[i];
+    }
+    const volumeFactor = 10 ** Math.log10(volume);
+    for (let i = 0; i < float32Data.length; i++) {
+      float32Data[i] *= volumeFactor;
+    }
+    const outputData = new Int16Array(float32Data.length);
+    for (let i = 0; i < float32Data.length; i++) {
+      const clipped = Math.max(-32768, Math.min(32767, float32Data[i]));
+      outputData[i] = Math.round(clipped);
+    }
+    return new AudioFrame(outputData, frame.sampleRate, frame.channels, frame.samplesPerChannel);
+  }
+  async playTask({
+    playHandle,
+    sound,
+    volume,
+    loop,
+    signal
+  }) {
+    if (isBuiltinAudioClip(sound)) {
+      sound = getBuiltinAudioPath(sound);
+    }
+    let audioStream;
+    if (typeof sound === "string") {
+      audioStream = loop ? loopAudioFramesFromFile(sound, { abortSignal: signal }) : audioFramesFromFile(sound, { abortSignal: signal });
+    } else {
+      audioStream = sound;
+    }
+    const applyVolume = this.applyVolumeToFrame.bind(this);
+    async function* genWrapper() {
+      for await (const frame of audioStream) {
+        if (signal.aborted || playHandle.done()) break;
+        yield volume !== 1 ? applyVolume(frame, volume) : frame;
+      }
+      playHandle._markPlayoutDone();
+    }
+    const gen = genWrapper();
+    try {
+      this.audioMixer.addStream(gen);
+      await playHandle.waitForPlayout();
+    } finally {
+      this.audioMixer.removeStream(gen);
+      playHandle._markPlayoutDone();
+      if (playHandle.done()) {
+        await gen.return(void 0);
+      }
+    }
+  }
+}
+export {
+  BackgroundAudioPlayer,
+  BuiltinAudioClip,
+  PlayHandle,
+  getBuiltinAudioPath,
+  isBuiltinAudioClip
+};
+//# sourceMappingURL=background_audio.js.map

package/dist/voice/background_audio.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/voice/background_audio.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AudioFrame,\n  AudioMixer,\n  AudioSource,\n  LocalAudioTrack,\n  type LocalTrackPublication,\n  type Room,\n  TrackPublishOptions,\n} from '@livekit/rtc-node';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { audioFramesFromFile, loopAudioFramesFromFile } from '../audio.js';\nimport { log } from '../log.js';\nimport { Future, Task, cancelAndWait } from '../utils.js';\nimport type { AgentSession } from './agent_session.js';\nimport { AgentSessionEventTypes, type AgentStateChangedEvent } from './events.js';\n\nconst TASK_TIMEOUT_MS = 500;\n\nexport enum BuiltinAudioClip {\n  OFFICE_AMBIENCE = 'office-ambience.ogg',\n  KEYBOARD_TYPING = 'keyboard-typing.ogg',\n  KEYBOARD_TYPING2 = 'keyboard-typing2.ogg',\n}\n\nexport function isBuiltinAudioClip(\n  source: AudioSourceType | AudioConfig | AudioConfig[],\n): source is BuiltinAudioClip {\n  return (\n    typeof source === 'string' &&\n    Object.values(BuiltinAudioClip).includes(source as BuiltinAudioClip)\n  );\n}\n\nexport function getBuiltinAudioPath(clip: BuiltinAudioClip): string {\n  const resourcesPath = join(dirname(fileURLToPath(import.meta.url)), '../../resources');\n  return join(resourcesPath, clip);\n}\n\nexport type AudioSourceType = string | BuiltinAudioClip | AsyncIterable<AudioFrame>;\n\nexport interface AudioConfig {\n  source: AudioSourceType;\n  volume?: number;\n  probability?: number;\n}\n\nexport interface BackgroundAudioPlayerOptions {\n  /**\n   * Ambient sound to play continuously in the background.\n   * Can be a file path, BuiltinAudioClip, or AudioConfig.\n   * File paths will be looped automatically.\n   */\n  ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  /**\n   * Sound to play when the agent is thinking.\n   * Plays when agent state changes to 'thinking' and stops when it changes to other states.\n   */\n  thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n  /**\n   * Stream timeout in milliseconds for the audio mixer.\n   * Controls how long the mixer waits for a stream to produce data before timing out.\n   * Higher values are more tolerant of network latency and processing delays.\n   * @defaultValue 2000\n   */\n  streamTimeoutMs?: number;\n}\n\nexport interface BackgroundAudioStartOptions {\n  room: Room;\n  agentSession?: AgentSession;\n  trackPublishOptions?: TrackPublishOptions;\n}\n\n// Queue size for AudioSource buffer (400ms)\n// Kept small to avoid abrupt cutoffs when removing sounds\nconst AUDIO_SOURCE_BUFFER_MS = 400;\nconst STREAM_TIMEOUT_MS = 2000;\n\nexport class PlayHandle {\n  private doneFuture = new Future<void>();\n  private stopFuture = new Future<void>();\n\n  done(): boolean {\n    return this.doneFuture.done;\n  }\n\n  stop(): void {\n    if (this.done()) return;\n\n    if (!this.stopFuture.done) {\n      this.stopFuture.resolve();\n    }\n\n    this._markPlayoutDone();\n  }\n\n  async waitForPlayout(): Promise<void> {\n    return this.doneFuture.await;\n  }\n\n  _markPlayoutDone(): void {\n    if (!this.doneFuture.done) {\n      this.doneFuture.resolve();\n    }\n  }\n}\n\n/**\n * Manages background audio playback for LiveKit agent sessions\n *\n * This class handles playing ambient sounds and manages audio track publishing.\n * It supports:\n * - Continuous ambient sound playback with looping\n * - Thinking sound playback during agent processing\n * - Multiple simultaneous audio streams via AudioMixer\n * - Volume control and probability-based sound selection\n * - Integration with LiveKit rooms and agent sessions\n *\n * @example\n * ```typescript\n * const player = new BackgroundAudioPlayer({\n *   ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },\n *   thinkingSound: { source: BuiltinAudioClip.KEYBOARD_TYPING, volume: 0.6 },\n * });\n *\n * await player.start({ room, agentSession });\n * ```\n */\nexport class BackgroundAudioPlayer {\n  private ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n  private thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n  private streamTimeoutMs: number;\n\n  private playTasks: Task<void>[] = [];\n  private audioSource = new AudioSource(48000, 1, AUDIO_SOURCE_BUFFER_MS);\n  private audioMixer: AudioMixer;\n  private mixerTask?: Task<void>;\n\n  private room?: Room;\n  private agentSession?: AgentSession;\n  private publication?: LocalTrackPublication;\n  private trackPublishOptions?: TrackPublishOptions;\n  private republishTask?: Task<void>;\n\n  private ambientHandle?: PlayHandle;\n  private thinkingHandle?: PlayHandle;\n\n  private closed = true;\n\n  // TODO (Brian): add lock\n\n  #logger = log();\n\n  constructor(options?: BackgroundAudioPlayerOptions) {\n    const { ambientSound, thinkingSound, streamTimeoutMs = STREAM_TIMEOUT_MS } = options || {};\n\n    this.ambientSound = ambientSound;\n    this.thinkingSound = thinkingSound;\n    this.streamTimeoutMs = streamTimeoutMs;\n\n    this.audioMixer = new AudioMixer(48000, 1, {\n      blocksize: 4800, // 100ms at 48kHz\n      capacity: 1,\n      streamTimeoutMs: this.streamTimeoutMs,\n    });\n  }\n\n  /**\n   * Select a sound from a list of background sound based on probability weights\n   * Return undefined if no sound is selected (when sum of probabilities < 1.0).\n   */\n  private selectSoundFromList(sounds: AudioConfig[]): AudioConfig | undefined {\n    const totalProbability = sounds.reduce((sum, sound) => sum + (sound.probability ?? 1.0), 0);\n\n    if (totalProbability <= 0) {\n      return undefined;\n    }\n\n    if (totalProbability < 1.0 && Math.random() > totalProbability) {\n      return undefined;\n    }\n\n    const normalizeFactor = totalProbability <= 1.0 ? 1.0 : totalProbability;\n    const r = Math.random() * Math.min(totalProbability, 1.0);\n    let cumulative = 0.0;\n\n    for (const sound of sounds) {\n      const prob = sound.probability ?? 1.0;\n      if (prob <= 0) {\n        continue;\n      }\n\n      const normProb = prob / normalizeFactor;\n      cumulative += normProb;\n\n      if (r <= cumulative) {\n        return sound;\n      }\n    }\n\n    return sounds[sounds.length - 1];\n  }\n\n  private normalizeSoundSource(\n    source?: AudioSourceType | AudioConfig | AudioConfig[],\n  ): { source: AudioSourceType; volume: number } | undefined {\n    if (source === undefined) {\n      return undefined;\n    }\n\n    if (typeof source === 'string') {\n      return {\n        source: this.normalizeBuiltinAudio(source),\n        volume: 1.0,\n      };\n    }\n\n    if (Array.isArray(source)) {\n      const selected = this.selectSoundFromList(source);\n      if (selected === undefined) {\n        return undefined;\n      }\n\n      return {\n        source: selected.source,\n        volume: selected.volume ?? 1.0,\n      };\n    }\n\n    if (typeof source === 'object' && 'source' in source) {\n      return {\n        source: this.normalizeBuiltinAudio(source.source),\n        volume: source.volume ?? 1.0,\n      };\n    }\n\n    return { source, volume: 1.0 };\n  }\n\n  private normalizeBuiltinAudio(source: AudioSourceType): AudioSourceType {\n    if (isBuiltinAudioClip(source)) {\n      return getBuiltinAudioPath(source);\n    }\n    return source;\n  }\n\n  play(audio: AudioSourceType | AudioConfig | AudioConfig[], loop = false): PlayHandle {\n    const normalized = this.normalizeSoundSource(audio);\n    if (normalized === undefined) {\n      const handle = new PlayHandle();\n      handle._markPlayoutDone();\n      return handle;\n    }\n\n    const { source, volume } = normalized;\n    const playHandle = new PlayHandle();\n\n    const task = Task.from(async ({ signal }) => {\n      await this.playTask({ playHandle, sound: source, volume, loop, signal });\n    });\n\n    task.addDoneCallback(() => {\n      playHandle._markPlayoutDone();\n      this.playTasks.splice(this.playTasks.indexOf(task), 1);\n    });\n\n    this.playTasks.push(task);\n    return playHandle;\n  }\n\n  /**\n   * Start the background audio system, publishing the audio track\n   * and beginning playback of any configured ambient sound.\n   *\n   * If `ambientSound` is provided (and contains file paths), they will loop\n   * automatically. If `ambientSound` contains AsyncIterators, they are assumed\n   * to be already infinite or looped.\n   *\n   * @param options - Options for starting background audio playback\n   */\n  async start(options: BackgroundAudioStartOptions): Promise<void> {\n    const { room, agentSession, trackPublishOptions } = options;\n    this.room = room;\n    this.agentSession = agentSession;\n    this.trackPublishOptions = trackPublishOptions;\n\n    this.closed = false;\n\n    await this.publishTrack();\n\n    // TODO (Brian): check job context is not fake\n\n    this.mixerTask = Task.from(async () => {\n      try {\n        await this.runMixerTask();\n      } catch (err) {\n        if (this.closed) return; // expected when AudioSource is closed\n        throw err;\n      }\n    });\n\n    this.room.on('reconnected', this.onReconnected);\n\n    this.agentSession?.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n    if (!this.ambientSound) return;\n\n    const normalized = this.normalizeSoundSource(this.ambientSound);\n    if (!normalized) return;\n\n    const { source, volume } = normalized;\n    const selectedSound: AudioConfig = { source, volume, probability: 1.0 };\n    this.ambientHandle = this.play(selectedSound, typeof source === 'string');\n  }\n\n  /**\n   * Close and cleanup the background audio system\n   */\n  async close(): Promise<void> {\n    this.closed = true;\n\n    await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);\n\n    if (this.republishTask) {\n      await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);\n    }\n\n    await this.audioMixer.aclose();\n    await this.audioSource.close();\n\n    if (this.mixerTask) {\n      await this.mixerTask.cancelAndWait(TASK_TIMEOUT_MS);\n    }\n\n    this.agentSession?.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n    this.room?.off('reconnected', this.onReconnected);\n\n    if (this.publication && this.publication.sid) {\n      await this.room?.localParticipant?.unpublishTrack(this.publication.sid);\n    }\n  }\n\n  /**\n   * Get the current track publication\n   */\n  getPublication(): LocalTrackPublication | undefined {\n    return this.publication;\n  }\n\n  private async publishTrack(): Promise<void> {\n    if (this.publication !== undefined) {\n      return;\n    }\n\n    const track = LocalAudioTrack.createAudioTrack('background_audio', this.audioSource);\n\n    if (this.room?.localParticipant === undefined) {\n      throw new Error('Local participant not available');\n    }\n\n    const publication = await this.room.localParticipant.publishTrack(\n      track,\n      this.trackPublishOptions ?? new TrackPublishOptions(),\n    );\n\n    this.publication = publication;\n    this.#logger.debug(`Background audio track published: ${this.publication.sid}`);\n  }\n\n  private onReconnected = (): void => {\n    if (this.republishTask) {\n      this.republishTask.cancel();\n    }\n\n    this.publication = undefined;\n    this.republishTask = Task.from(async () => {\n      await this.republishTrackTask();\n    });\n  };\n\n  private async republishTrackTask(): Promise<void> {\n    // TODO (Brian): add lock protection when implementing lock\n    await this.publishTrack();\n  }\n\n  private async runMixerTask(): Promise<void> {\n    for await (const frame of this.audioMixer) {\n      await this.audioSource.captureFrame(frame);\n    }\n  }\n\n  private onAgentStateChanged = (ev: AgentStateChangedEvent): void => {\n    if (!this.thinkingSound) {\n      return;\n    }\n\n    if (ev.newState === 'thinking') {\n      if (this.thinkingHandle && !this.thinkingHandle.done()) {\n        return;\n      }\n\n      const normalized = this.normalizeSoundSource(this.thinkingSound);\n      if (normalized) {\n        const { source, volume } = normalized;\n        const selectedSound: AudioConfig = { source, volume, probability: 1.0 };\n        // Loop thinking sound while in thinking state (same as ambient)\n        this.thinkingHandle = this.play(selectedSound, typeof source === 'string');\n      }\n    } else {\n      this.thinkingHandle?.stop();\n    }\n  };\n\n  // Note: Python uses numpy, TS uses typed arrays for equivalent logic\n  private applyVolumeToFrame(frame: AudioFrame, volume: number): AudioFrame {\n    const int16Data = new Int16Array(\n      frame.data.buffer,\n      frame.data.byteOffset,\n      frame.data.byteLength / 2,\n    );\n    const float32Data = new Float32Array(int16Data.length);\n\n    for (let i = 0; i < int16Data.length; i++) {\n      float32Data[i] = int16Data[i]!;\n    }\n\n    const volumeFactor = 10 ** Math.log10(volume);\n    for (let i = 0; i < float32Data.length; i++) {\n      float32Data[i]! *= volumeFactor;\n    }\n\n    const outputData = new Int16Array(float32Data.length);\n    for (let i = 0; i < float32Data.length; i++) {\n      const clipped = Math.max(-32768, Math.min(32767, float32Data[i]!));\n      outputData[i] = Math.round(clipped);\n    }\n\n    return new AudioFrame(outputData, frame.sampleRate, frame.channels, frame.samplesPerChannel);\n  }\n\n  private async playTask({\n    playHandle,\n    sound,\n    volume,\n    loop,\n    signal,\n  }: {\n    playHandle: PlayHandle;\n    sound: AudioSourceType;\n    volume: number;\n    loop: boolean;\n    signal: AbortSignal;\n  }): Promise<void> {\n    if (isBuiltinAudioClip(sound)) {\n      sound = getBuiltinAudioPath(sound);\n    }\n\n    let audioStream: AsyncIterable<AudioFrame>;\n    if (typeof sound === 'string') {\n      audioStream = loop\n        ? loopAudioFramesFromFile(sound, { abortSignal: signal })\n        : audioFramesFromFile(sound, { abortSignal: signal });\n    } else {\n      audioStream = sound;\n    }\n\n    const applyVolume = this.applyVolumeToFrame.bind(this);\n    async function* genWrapper(): AsyncGenerator<AudioFrame> {\n      for await (const frame of audioStream) {\n        if (signal.aborted || playHandle.done()) break;\n        yield volume !== 1.0 ? applyVolume(frame, volume) : frame;\n      }\n      playHandle._markPlayoutDone();\n    }\n\n    const gen = genWrapper();\n    try {\n      this.audioMixer.addStream(gen);\n      await playHandle.waitForPlayout();\n    } finally {\n      this.audioMixer.removeStream(gen);\n      playHandle._markPlayoutDone();\n\n      if (playHandle.done()) {\n        await gen.return(undefined);\n      }\n    }\n  }\n}\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAGA;AAAA,OACK;AACP,SAAS,SAAS,YAAY;AAC9B,SAAS,qBAAqB;AAC9B,SAAS,qBAAqB,+BAA+B;AAC7D,SAAS,WAAW;AACpB,SAAS,QAAQ,MAAM,qBAAqB;AAE5C,SAAS,8BAA2D;AAEpE,MAAM,kBAAkB;AAEjB,IAAK,mBAAL,kBAAKA,sBAAL;AACL,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,sBAAmB;AAHT,SAAAA;AAAA,GAAA;AAML,SAAS,mBACd,QAC4B;AAC5B,SACE,OAAO,WAAW,YAClB,OAAO,OAAO,gBAAgB,EAAE,SAAS,MAA0B;AAEvE;AAEO,SAAS,oBAAoB,MAAgC;AAClE,QAAM,gBAAgB,KAAK,QAAQ,cAAc,YAAY,GAAG,CAAC,GAAG,iBAAiB;AACrF,SAAO,KAAK,eAAe,IAAI;AACjC;AAyCA,MAAM,yBAAyB;AAC/B,MAAM,oBAAoB;AAEnB,MAAM,WAAW;AAAA,EACd,aAAa,IAAI,OAAa;AAAA,EAC9B,aAAa,IAAI,OAAa;AAAA,EAEtC,OAAgB;AACd,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,OAAa;AACX,QAAI,KAAK,KAAK,EAAG;AAEjB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,MAAM,iBAAgC;AACpC,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,mBAAyB;AACvB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAAA,EACF;AACF;AAuBO,MAAM,sBAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAA0B,CAAC;AAAA,EAC3B,cAAc,IAAI,YAAY,MAAO,GAAG,sBAAsB;AAAA,EAC9D;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,SAAS;AAAA;AAAA,EAIjB,UAAU,IAAI;AAAA,EAEd,YAAY,SAAwC;AAClD,UAAM,EAAE,cAAc,eAAe,kBAAkB,kBAAkB,IAAI,WAAW,CAAC;AAEzF,SAAK,eAAe;AACpB,SAAK,gBAAgB;AACrB,SAAK,kBAAkB;AAEvB,SAAK,aAAa,IAAI,WAAW,MAAO,GAAG;AAAA,MACzC,WAAW;AAAA;AAAA,MACX,UAAU;AAAA,MACV,iBAAiB,KAAK;AAAA,IACxB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,oBAAoB,QAAgD;AAC1E,UAAM,mBAAmB,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,MAAM,eAAe,IAAM,CAAC;AAE1F,QAAI,oBAAoB,GAAG;AACzB,aAAO;AAAA,IACT;AAEA,QAAI,mBAAmB,KAAO,KAAK,OAAO,IAAI,kBAAkB;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,oBAAoB,IAAM,IAAM;AACxD,UAAM,IAAI,KAAK,OAAO,IAAI,KAAK,IAAI,kBAAkB,CAAG;AACxD,QAAI,aAAa;AAEjB,eAAW,SAAS,QAAQ;AAC1B,YAAM,OAAO,MAAM,eAAe;AAClC,UAAI,QAAQ,GAAG;AACb;AAAA,MACF;AAEA,YAAM,WAAW,OAAO;AACxB,oBAAc;AAEd,UAAI,KAAK,YAAY;AACnB,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,OAAO,OAAO,SAAS,CAAC;AAAA,EACjC;AAAA,EAEQ,qBACN,QACyD;AACzD,QAAI,WAAW,QAAW;AACxB,aAAO;AAAA,IACT;AAEA,QAAI,OAAO,WAAW,UAAU;AAC9B,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,MAAM;AAAA,QACzC,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,YAAM,WAAW,KAAK,oBAAoB,MAAM;AAChD,UAAI,aAAa,QAAW;AAC1B,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL,QAAQ,SAAS;AAAA,QACjB,QAAQ,SAAS,UAAU;AAAA,MAC7B;AAAA,IACF;AAEA,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,OAAO,MAAM;AAAA,QAChD,QAAQ,OAAO,UAAU;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ,QAAQ,EAAI;AAAA,EAC/B;AAAA,EAEQ,sBAAsB,QAA0C;AACtE,QAAI,mBAAmB,MAAM,GAAG;AAC9B,aAAO,oBAAoB,MAAM;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,OAAsD,OAAO,OAAmB;AACnF,UAAM,aAAa,KAAK,qBAAqB,KAAK;AAClD,QAAI,eAAe,QAAW;AAC5B,YAAM,SAAS,IAAI,WAAW;AAC9B,aAAO,iBAAiB;AACxB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,aAAa,IAAI,WAAW;AAElC,UAAM,OAAO,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AAC3C,YAAM,KAAK,SAAS,EAAE,YAAY,OAAO,QAAQ,QAAQ,MAAM,OAAO,CAAC;AAAA,IACzE,CAAC;AAED,SAAK,gBAAgB,MAAM;AACzB,iBAAW,iBAAiB;AAC5B,WAAK,UAAU,OAAO,KAAK,UAAU,QAAQ,IAAI,GAAG,CAAC;AAAA,IACvD,CAAC;AAED,SAAK,UAAU,KAAK,IAAI;AACxB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,MAAM,SAAqD;AA9RnE;AA+RI,UAAM,EAAE,MAAM,cAAc,oBAAoB,IAAI;AACpD,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB;AAE3B,SAAK,SAAS;AAEd,UAAM,KAAK,aAAa;AAIxB,SAAK,YAAY,KAAK,KAAK,YAAY;AACrC,UAAI;AACF,cAAM,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK;AACZ,YAAI,KAAK,OAAQ;AACjB,cAAM;AAAA,MACR;AAAA,IACF,CAAC;AAED,SAAK,KAAK,GAAG,eAAe,KAAK,aAAa;AAE9C,eAAK,iBAAL,mBAAmB,GAAG,uBAAuB,mBAAmB,KAAK;AACrE,QAAI,CAAC,KAAK,aAAc;AAExB,UAAM,aAAa,KAAK,qBAAqB,KAAK,YAAY;AAC9D,QAAI,CAAC,WAAY;AAEjB,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,gBAA6B,EAAE,QAAQ,QAAQ,aAAa,EAAI;AACtE,SAAK,gBAAgB,KAAK,KAAK,eAAe,OAAO,WAAW,QAAQ;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAnU/B;AAoUI,SAAK,SAAS;AAEd,UAAM,cAAc,KAAK,WAAW,eAAe;AAEnD,QAAI,KAAK,eAAe;AACtB,YAAM,KAAK,cAAc,cAAc,eAAe;AAAA,IACxD;AAEA,UAAM,KAAK,WAAW,OAAO;AAC7B,UAAM,KAAK,YAAY,MAAM;AAE7B,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK,UAAU,cAAc,eAAe;AAAA,IACpD;AAEA,eAAK,iBAAL,mBAAmB,IAAI,uBAAuB,mBAAmB,KAAK;AACtE,eAAK,SAAL,mBAAW,IAAI,eAAe,KAAK;AAEnC,QAAI,KAAK,eAAe,KAAK,YAAY,KAAK;AAC5C,cAAM,gBAAK,SAAL,mBAAW,qBAAX,mBAA6B,eAAe,KAAK,YAAY;AAAA,IACrE;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAoD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,eAA8B;AAlW9C;AAmWI,QAAI,KAAK,gBAAgB,QAAW;AAClC;AAAA,IACF;AAEA,UAAM,QAAQ,gBAAgB,iBAAiB,oBAAoB,KAAK,WAAW;AAEnF,UAAI,UAAK,SAAL,mBAAW,sBAAqB,QAAW;AAC7C,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,UAAM,cAAc,MAAM,KAAK,KAAK,iBAAiB;AAAA,MACnD;AAAA,MACA,KAAK,uBAAuB,IAAI,oBAAoB;AAAA,IACtD;AAEA,SAAK,cAAc;AACnB,SAAK,QAAQ,MAAM,qCAAqC,KAAK,YAAY,GAAG,EAAE;AAAA,EAChF;AAAA,EAEQ,gBAAgB,MAAY;AAClC,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,SAAK,cAAc;AACnB,SAAK,gBAAgB,KAAK,KAAK,YAAY;AACzC,YAAM,KAAK,mBAAmB;AAAA,IAChC,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,qBAAoC;AAEhD,UAAM,KAAK,aAAa;AAAA,EAC1B;AAAA,EAEA,MAAc,eAA8B;AAC1C,qBAAiB,SAAS,KAAK,YAAY;AACzC,YAAM,KAAK,YAAY,aAAa,KAAK;AAAA,IAC3C;AAAA,EACF;AAAA,EAEQ,sBAAsB,CAAC,OAAqC;AA5YtE;AA6YI,QAAI,CAAC,KAAK,eAAe;AACvB;AAAA,IACF;AAEA,QAAI,GAAG,aAAa,YAAY;AAC9B,UAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,KAAK,GAAG;AACtD;AAAA,MACF;AAEA,YAAM,aAAa,KAAK,qBAAqB,KAAK,aAAa;AAC/D,UAAI,YAAY;AACd,cAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,cAAM,gBAA6B,EAAE,QAAQ,QAAQ,aAAa,EAAI;AAEtE,aAAK,iBAAiB,KAAK,KAAK,eAAe,OAAO,WAAW,QAAQ;AAAA,MAC3E;AAAA,IACF,OAAO;AACL,iBAAK,mBAAL,mBAAqB;AAAA,IACvB;AAAA,EACF;AAAA;AAAA,EAGQ,mBAAmB,OAAmB,QAA4B;AACxE,UAAM,YAAY,IAAI;AAAA,MACpB,MAAM,KAAK;AAAA,MACX,MAAM,KAAK;AAAA,MACX,MAAM,KAAK,aAAa;AAAA,IAC1B;AACA,UAAM,cAAc,IAAI,aAAa,UAAU,MAAM;AAErD,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,kBAAY,CAAC,IAAI,UAAU,CAAC;AAAA,IAC9B;AAEA,UAAM,eAAe,MAAM,KAAK,MAAM,MAAM;AAC5C,aAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,kBAAY,CAAC,KAAM;AAAA,IACrB;AAEA,UAAM,aAAa,IAAI,WAAW,YAAY,MAAM;AACpD,aAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,YAAM,UAAU,KAAK,IAAI,QAAQ,KAAK,IAAI,OAAO,YAAY,CAAC,CAAE,CAAC;AACjE,iBAAW,CAAC,IAAI,KAAK,MAAM,OAAO;AAAA,IACpC;AAEA,WAAO,IAAI,WAAW,YAAY,MAAM,YAAY,MAAM,UAAU,MAAM,iBAAiB;AAAA,EAC7F;AAAA,EAEA,MAAc,SAAS;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMkB;AAChB,QAAI,mBAAmB,KAAK,GAAG;AAC7B,cAAQ,oBAAoB,KAAK;AAAA,IACnC;AAEA,QAAI;AACJ,QAAI,OAAO,UAAU,UAAU;AAC7B,oBAAc,OACV,wBAAwB,OAAO,EAAE,aAAa,OAAO,CAAC,IACtD,oBAAoB,OAAO,EAAE,aAAa,OAAO,CAAC;AAAA,IACxD,OAAO;AACL,oBAAc;AAAA,IAChB;AAEA,UAAM,cAAc,KAAK,mBAAmB,KAAK,IAAI;AACrD,oBAAgB,aAAyC;AACvD,uBAAiB,SAAS,aAAa;AACrC,YAAI,OAAO,WAAW,WAAW,KAAK,EAAG;AACzC,cAAM,WAAW,IAAM,YAAY,OAAO,MAAM,IAAI;AAAA,MACtD;AACA,iBAAW,iBAAiB;AAAA,IAC9B;AAEA,UAAM,MAAM,WAAW;AACvB,QAAI;AACF,WAAK,WAAW,UAAU,GAAG;AAC7B,YAAM,WAAW,eAAe;AAAA,IAClC,UAAE;AACA,WAAK,WAAW,aAAa,GAAG;AAChC,iBAAW,iBAAiB;AAE5B,UAAI,WAAW,KAAK,GAAG;AACrB,cAAM,IAAI,OAAO,MAAS;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AACF;","names":["BuiltinAudioClip"]}

package/dist/voice/events.cjs

@@ -40,6 +40,7 @@ var AgentSessionEventTypes = /* @__PURE__ */ ((AgentSessionEventTypes2) => {
   AgentSessionEventTypes2["FunctionToolsExecuted"] = "function_tools_executed";
   AgentSessionEventTypes2["MetricsCollected"] = "metrics_collected";
   AgentSessionEventTypes2["SpeechCreated"] = "speech_created";
+  AgentSessionEventTypes2["OverlappingSpeech"] = "overlapping_speech";
   AgentSessionEventTypes2["Error"] = "error";
   AgentSessionEventTypes2["Close"] = "close";
   return AgentSessionEventTypes2;
@@ -67,12 +68,14 @@ const createUserInputTranscribedEvent = ({
   transcript,
   isFinal,
   speakerId = null,
+  language = null,
   createdAt = Date.now()
 }) => ({
   type: "user_input_transcribed",
   transcript,
   isFinal,
   speakerId,
+  language,
   createdAt
 });
 const createMetricsCollectedEvent = ({

package/dist/voice/events.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  speakerId: string | null;\n  createdAt: number;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAUO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,YAAY,KAAK,IAAI;AACvB,OAKkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}
+{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { InterruptionDetectionError } from '../inference/interruption/errors.js';\nimport type { OverlappingSpeechEvent } from '../inference/interruption/types.js';\nimport type { LanguageCode } from '../language.js';\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  LLM,\n  RealtimeModel,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  OverlappingSpeech = 'overlapping_speech',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type ShutdownReason = CloseReason | string;\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  // TODO(AJS-106): add multi participant support\n  /** Not supported yet. Always null by default. */\n  speakerId: string | null;\n  createdAt: number;\n  language: LanguageCode | null;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  language = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  language?: LanguageCode | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  language,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null;\n  reason: ShutdownReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: ShutdownReason,\n  error:\n    | RealtimeModelError\n    | STTError\n    | TTSError\n    | LLMError\n    | InterruptionDetectionError\n    | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | OverlappingSpeechEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAsBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AAVE,SAAAA;AAAA,GAAA;AAgBL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAkBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAaO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,YAAY,KAAK,IAAI;AACvB,OAMkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAMW,MACX,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}

package/dist/voice/events.d.cts

@@ -1,5 +1,7 @@
-import type { ChatMessage, FunctionCall, FunctionCallOutput, RealtimeModelError } from '../llm/index.js';
-import type { LLM, RealtimeModel } from '../llm/index.js';
+import type { InterruptionDetectionError } from '../inference/interruption/errors.js';
+import type { OverlappingSpeechEvent } from '../inference/interruption/types.js';
+import type { LanguageCode } from '../language.js';
+import type { ChatMessage, FunctionCall, FunctionCallOutput, LLM, RealtimeModel, RealtimeModelError } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import type { AgentMetrics } from '../metrics/base.js';
 import type { STT } from '../stt/index.js';
@@ -15,6 +17,7 @@ export declare enum AgentSessionEventTypes {
     FunctionToolsExecuted = "function_tools_executed",
     MetricsCollected = "metrics_collected",
     SpeechCreated = "speech_created",
+    OverlappingSpeech = "overlapping_speech",
     Error = "error",
     Close = "close"
 }
@@ -26,6 +29,7 @@ export declare enum CloseReason {
     PARTICIPANT_DISCONNECTED = "participant_disconnected",
     USER_INITIATED = "user_initiated"
 }
+export type ShutdownReason = CloseReason | string;
 export type SpeechSource = 'say' | 'generate_reply' | 'tool_response';
 export type UserStateChangedEvent = {
     type: 'user_state_changed';
@@ -45,13 +49,16 @@ export type UserInputTranscribedEvent = {
     type: 'user_input_transcribed';
     transcript: string;
     isFinal: boolean;
+    /** Not supported yet. Always null by default. */
     speakerId: string | null;
     createdAt: number;
+    language: LanguageCode | null;
 };
-export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, createdAt, }: {
+export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, language, createdAt, }: {
     transcript: string;
     isFinal: boolean;
     speakerId?: string | null;
+    language?: LanguageCode | null;
     createdAt?: number;
 }) => UserInputTranscribedEvent;
 export type MetricsCollectedEvent = {
@@ -108,17 +115,17 @@ export declare const createSpeechCreatedEvent: ({ userInitiated, source, speechH
 }) => SpeechCreatedEvent;
 export type ErrorEvent = {
     type: 'error';
-    error: RealtimeModelError | STTError | TTSError | LLMError | unknown;
+    error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown;
     source: LLM | STT | TTS | RealtimeModel | unknown;
     createdAt: number;
 };
-export declare const createErrorEvent: (error: RealtimeModelError | STTError | TTSError | LLMError | unknown, source: LLM | STT | TTS | RealtimeModel | unknown, createdAt?: number) => ErrorEvent;
+export declare const createErrorEvent: (error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown, source: LLM | STT | TTS | RealtimeModel | unknown, createdAt?: number) => ErrorEvent;
 export type CloseEvent = {
     type: 'close';
-    error: RealtimeModelError | STTError | TTSError | LLMError | null;
-    reason: CloseReason;
+    error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null;
+    reason: ShutdownReason;
     createdAt: number;
 };
-export declare const createCloseEvent: (reason: CloseReason, error?: RealtimeModelError | STTError | TTSError | LLMError | null, createdAt?: number) => CloseEvent;
-export type AgentEvent = UserInputTranscribedEvent | UserStateChangedEvent | AgentStateChangedEvent | MetricsCollectedEvent | ConversationItemAddedEvent | FunctionToolsExecutedEvent | SpeechCreatedEvent | ErrorEvent | CloseEvent;
+export declare const createCloseEvent: (reason: ShutdownReason, error?: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null, createdAt?: number) => CloseEvent;
+export type AgentEvent = UserInputTranscribedEvent | UserStateChangedEvent | AgentStateChangedEvent | MetricsCollectedEvent | ConversationItemAddedEvent | FunctionToolsExecutedEvent | SpeechCreatedEvent | OverlappingSpeechEvent | ErrorEvent | CloseEvent;
 //# sourceMappingURL=events.d.ts.map

package/dist/voice/events.d.ts

@@ -1,5 +1,7 @@
-import type { ChatMessage, FunctionCall, FunctionCallOutput, RealtimeModelError } from '../llm/index.js';
-import type { LLM, RealtimeModel } from '../llm/index.js';
+import type { InterruptionDetectionError } from '../inference/interruption/errors.js';
+import type { OverlappingSpeechEvent } from '../inference/interruption/types.js';
+import type { LanguageCode } from '../language.js';
+import type { ChatMessage, FunctionCall, FunctionCallOutput, LLM, RealtimeModel, RealtimeModelError } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import type { AgentMetrics } from '../metrics/base.js';
 import type { STT } from '../stt/index.js';
@@ -15,6 +17,7 @@ export declare enum AgentSessionEventTypes {
     FunctionToolsExecuted = "function_tools_executed",
     MetricsCollected = "metrics_collected",
     SpeechCreated = "speech_created",
+    OverlappingSpeech = "overlapping_speech",
     Error = "error",
     Close = "close"
 }
@@ -26,6 +29,7 @@ export declare enum CloseReason {
     PARTICIPANT_DISCONNECTED = "participant_disconnected",
     USER_INITIATED = "user_initiated"
 }
+export type ShutdownReason = CloseReason | string;
 export type SpeechSource = 'say' | 'generate_reply' | 'tool_response';
 export type UserStateChangedEvent = {
     type: 'user_state_changed';
@@ -45,13 +49,16 @@ export type UserInputTranscribedEvent = {
     type: 'user_input_transcribed';
     transcript: string;
     isFinal: boolean;
+    /** Not supported yet. Always null by default. */
     speakerId: string | null;
     createdAt: number;
+    language: LanguageCode | null;
 };
-export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, createdAt, }: {
+export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, language, createdAt, }: {
     transcript: string;
     isFinal: boolean;
     speakerId?: string | null;
+    language?: LanguageCode | null;
     createdAt?: number;
 }) => UserInputTranscribedEvent;
 export type MetricsCollectedEvent = {
@@ -108,17 +115,17 @@ export declare const createSpeechCreatedEvent: ({ userInitiated, source, speechH
 }) => SpeechCreatedEvent;
 export type ErrorEvent = {
     type: 'error';
-    error: RealtimeModelError | STTError | TTSError | LLMError | unknown;
+    error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown;
     source: LLM | STT | TTS | RealtimeModel | unknown;
     createdAt: number;
 };
-export declare const createErrorEvent: (error: RealtimeModelError | STTError | TTSError | LLMError | unknown, source: LLM | STT | TTS | RealtimeModel | unknown, createdAt?: number) => ErrorEvent;
+export declare const createErrorEvent: (error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown, source: LLM | STT | TTS | RealtimeModel | unknown, createdAt?: number) => ErrorEvent;
 export type CloseEvent = {
     type: 'close';
-    error: RealtimeModelError | STTError | TTSError | LLMError | null;
-    reason: CloseReason;
+    error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null;
+    reason: ShutdownReason;
     createdAt: number;
 };
-export declare const createCloseEvent: (reason: CloseReason, error?: RealtimeModelError | STTError | TTSError | LLMError | null, createdAt?: number) => CloseEvent;
-export type AgentEvent = UserInputTranscribedEvent | UserStateChangedEvent | AgentStateChangedEvent | MetricsCollectedEvent | ConversationItemAddedEvent | FunctionToolsExecutedEvent | SpeechCreatedEvent | ErrorEvent | CloseEvent;
+export declare const createCloseEvent: (reason: ShutdownReason, error?: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null, createdAt?: number) => CloseEvent;
+export type AgentEvent = UserInputTranscribedEvent | UserStateChangedEvent | AgentStateChangedEvent | MetricsCollectedEvent | ConversationItemAddedEvent | FunctionToolsExecutedEvent | SpeechCreatedEvent | OverlappingSpeechEvent | ErrorEvent | CloseEvent;
 //# sourceMappingURL=events.d.ts.map

package/dist/voice/events.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/voice/events.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,oBAAY,sBAAsB;IAChC,oBAAoB,2BAA2B;IAC/C,iBAAiB,wBAAwB;IACzC,gBAAgB,uBAAuB;IACvC,qBAAqB,4BAA4B;IACjD,qBAAqB,4BAA4B;IACjD,gBAAgB,sBAAsB;IACtC,aAAa,mBAAmB;IAChC,KAAK,UAAU;IACf,KAAK,UAAU;CAChB;AAED,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,MAAM,CAAC;AAC1D,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAEzF,oBAAY,WAAW;IACrB,KAAK,UAAU;IACf,YAAY,iBAAiB;IAC7B,wBAAwB,6BAA6B;IACrD,cAAc,mBAAmB;CAClC;AAED,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEtE,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,QAAQ,EAAE,SAAS,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAC5B,SAAS,YACT,SAAS,cACR,MAAM,KAChB,qBAKD,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,qBAAqB,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC;IACrB,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,4BAA4B,aAC7B,UAAU,YACV,UAAU,cACT,MAAM,KAChB,sBAKD,CAAC;AAEH,MAAM,MAAM,yBAAyB,GAAG;IACtC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,+BAA+B,mDAKzC;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,yBAMF,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,4BAGrC;IACD,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,qBAIF,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,SACrC,WAAW,cACN,MAAM,KAChB,0BAID,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,uDAI1C;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,0BAOH,CAAC;AAEF,eAAO,MAAM,0BAA0B,UAC9B,0BAA0B,KAChC,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAE1C,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB;;OAEG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;OAEG;IAEH,YAAY,EAAE,YAAY,CAAC;IAC3B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,wBAAwB,wDAKlC;IACD,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,kBAMF,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACrE,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,UACpB,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,UAC5D,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,cACtC,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClE,MAAM,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,WACnB,WAAW,UACZ,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,cACtD,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAClB,yBAAyB,GACzB,qBAAqB,GACrB,sBAAsB,GACtB,qBAAqB,GACrB,0BAA0B,GAC1B,0BAA0B,GAC1B,kBAAkB,GAClB,UAAU,GACV,UAAU,CAAC"}
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/voice/events.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,qCAAqC,CAAC;AACtF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,GAAG,EACH,aAAa,EACb,kBAAkB,EACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,oBAAY,sBAAsB;IAChC,oBAAoB,2BAA2B;IAC/C,iBAAiB,wBAAwB;IACzC,gBAAgB,uBAAuB;IACvC,qBAAqB,4BAA4B;IACjD,qBAAqB,4BAA4B;IACjD,gBAAgB,sBAAsB;IACtC,aAAa,mBAAmB;IAChC,iBAAiB,uBAAuB;IACxC,KAAK,UAAU;IACf,KAAK,UAAU;CAChB;AAED,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,MAAM,CAAC;AAC1D,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAEzF,oBAAY,WAAW;IACrB,KAAK,UAAU;IACf,YAAY,iBAAiB;IAC7B,wBAAwB,6BAA6B;IACrD,cAAc,mBAAmB;CAClC;AAED,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,MAAM,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEtE,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,QAAQ,EAAE,SAAS,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAC5B,SAAS,YACT,SAAS,cACR,MAAM,KAChB,qBAKD,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,qBAAqB,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC;IACrB,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,4BAA4B,aAC7B,UAAU,YACV,UAAU,cACT,MAAM,KAChB,sBAKD,CAAC;AAEH,MAAM,MAAM,yBAAyB,GAAG;IACtC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IAEjB,iDAAiD;IACjD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,YAAY,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,+BAA+B,6DAMzC;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,yBAOF,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,4BAGrC;IACD,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,qBAIF,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,SACrC,WAAW,cACN,MAAM,KAChB,0BAID,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,uDAI1C;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,0BAOH,CAAC;AAEF,eAAO,MAAM,0BAA0B,UAC9B,0BAA0B,KAChC,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAE1C,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB;;OAEG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;OAEG;IAEH,YAAY,EAAE,YAAY,CAAC;IAC3B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,wBAAwB,wDAKlC;IACD,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,kBAMF,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,0BAA0B,GAAG,OAAO,CAAC;IAClG,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,UACpB,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,0BAA0B,GAAG,OAAO,UACzF,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,cACtC,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,0BAA0B,GAAG,IAAI,CAAC;IAC/F,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,WACnB,cAAc,UAElB,kBAAkB,GAClB,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,0BAA0B,GAC1B,IAAI,cACG,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAClB,yBAAyB,GACzB,qBAAqB,GACrB,sBAAsB,GACtB,qBAAqB,GACrB,0BAA0B,GAC1B,0BAA0B,GAC1B,kBAAkB,GAClB,sBAAsB,GACtB,UAAU,GACV,UAAU,CAAC"}

package/dist/voice/events.js

@@ -6,6 +6,7 @@ var AgentSessionEventTypes = /* @__PURE__ */ ((AgentSessionEventTypes2) => {
   AgentSessionEventTypes2["FunctionToolsExecuted"] = "function_tools_executed";
   AgentSessionEventTypes2["MetricsCollected"] = "metrics_collected";
   AgentSessionEventTypes2["SpeechCreated"] = "speech_created";
+  AgentSessionEventTypes2["OverlappingSpeech"] = "overlapping_speech";
   AgentSessionEventTypes2["Error"] = "error";
   AgentSessionEventTypes2["Close"] = "close";
   return AgentSessionEventTypes2;
@@ -33,12 +34,14 @@ const createUserInputTranscribedEvent = ({
   transcript,
   isFinal,
   speakerId = null,
+  language = null,
   createdAt = Date.now()
 }) => ({
   type: "user_input_transcribed",
   transcript,
   isFinal,
   speakerId,
+  language,
   createdAt
 });
 const createMetricsCollectedEvent = ({