@livekit/agents 1.1.0 → 1.2.0

package/src/voice/turn_config/endpointing.ts

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Configuration for endpointing, which determines when the user's turn is complete.
+ */
+export interface EndpointingOptions {
+  /**
+   * Endpointing mode. `"fixed"` uses a fixed delay, `"dynamic"` adjusts delay based on
+   * end-of-utterance prediction.
+   * @defaultValue "fixed"
+   */
+  mode: 'fixed' | 'dynamic';
+  /**
+   * Minimum time in milliseconds since the last detected speech before the agent declares the user's
+   * turn complete. In VAD mode this effectively behaves like `max(VAD silence, minDelay)`;
+   * in STT mode it is applied after the STT end-of-speech signal, so it can be additive with
+   * the STT provider's endpointing delay.
+   * @defaultValue 500
+   */
+  minDelay: number;
+  /**
+   * Maximum time in milliseconds the agent will wait before terminating the turn.
+   * @defaultValue 3000
+   */
+  maxDelay: number;
+}
+
+export const defaultEndpointingOptions = {
+  mode: 'fixed',
+  minDelay: 500,
+  maxDelay: 3000,
+} as const satisfies EndpointingOptions;

package/src/voice/turn_config/interruption.ts

@@ -0,0 +1,56 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Configuration for interruption handling.
+ */
+export interface InterruptionOptions {
+  /**
+   * Whether interruptions are enabled.
+   * @defaultValue true
+   */
+  enabled: boolean;
+  /**
+   * Interruption handling strategy. `"adaptive"` for ML-based detection, `"vad"` for simple
+   * voice-activity detection. `undefined` means auto-detect.
+   * @defaultValue undefined
+   */
+  mode: 'adaptive' | 'vad' | undefined;
+  /**
+   * When `true`, buffered audio is dropped while the agent is speaking and cannot be interrupted.
+   * @defaultValue true
+   */
+  discardAudioIfUninterruptible: boolean;
+  /**
+   * Minimum speech length in milliseconds to register as an interruption.
+   * @defaultValue 500
+   */
+  minDuration: number;
+  /**
+   * Minimum number of words to consider an interruption, only used if STT is enabled.
+   * @defaultValue 0
+   */
+  minWords: number;
+  /**
+   * If set, emit an `agentFalseInterruption` event after this amount of time if the user is
+   * silent and no user transcript is detected after the interruption. Set to `undefined` to
+   * disable. The value is in milliseconds.
+   * @defaultValue 2000
+   */
+  falseInterruptionTimeout: number;
+  /**
+   * Whether to resume the false interruption after the `falseInterruptionTimeout`.
+   * @defaultValue true
+   */
+  resumeFalseInterruption: boolean;
+}
+
+export const defaultInterruptionOptions = {
+  enabled: true,
+  mode: undefined,
+  discardAudioIfUninterruptible: true,
+  minDuration: 500,
+  minWords: 0,
+  falseInterruptionTimeout: 2000,
+  resumeFalseInterruption: true,
+} as const satisfies InterruptionOptions;

package/src/voice/turn_config/turn_handling.ts

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { TurnDetectionMode } from '../agent_session.js';
+import { type EndpointingOptions, defaultEndpointingOptions } from './endpointing.js';
+import { type InterruptionOptions, defaultInterruptionOptions } from './interruption.js';
+
+/**
+ * Configuration for the turn handling system. Used to configure the turn taking behavior of the
+ * session.
+ */
+export interface TurnHandlingOptions {
+  /**
+   * Strategy for deciding when the user has finished speaking.
+   *
+   * - `"stt"` – rely on speech-to-text end-of-utterance cues
+   * - `"vad"` – rely on Voice Activity Detection start/stop cues
+   * - `"realtime_llm"` – use server-side detection from a realtime LLM
+   * - `"manual"` – caller controls turn boundaries explicitly
+   *
+   * If not set, the session chooses the best available mode in priority order
+   * `realtime_llm → vad → stt → manual`; it automatically falls back if the necessary model
+   * is missing.
+   */
+  turnDetection: TurnDetectionMode | undefined;
+  /**
+   * Configuration for endpointing.
+   */
+  endpointing: Partial<EndpointingOptions>;
+  /**
+   * Configuration for interruption handling.
+   */
+  interruption: Partial<InterruptionOptions>;
+}
+
+export interface InternalTurnHandlingOptions extends TurnHandlingOptions {
+  endpointing: EndpointingOptions;
+  interruption: InterruptionOptions;
+}
+
+export const defaultTurnHandlingOptions: InternalTurnHandlingOptions = {
+  turnDetection: undefined,
+  interruption: defaultInterruptionOptions,
+  endpointing: defaultEndpointingOptions,
+};

package/src/voice/turn_config/utils.test.ts

@@ -0,0 +1,148 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { beforeAll, describe, expect, it } from 'vitest';
+import { initializeLogger } from '../../log.js';
+import { defaultAgentSessionOptions } from '../agent_session.js';
+import { defaultEndpointingOptions } from './endpointing.js';
+import { defaultInterruptionOptions } from './interruption.js';
+import { defaultTurnHandlingOptions } from './turn_handling.js';
+import { migrateLegacyOptions, migrateTurnHandling } from './utils.js';
+
+beforeAll(() => {
+  initializeLogger({ pretty: true, level: 'info' });
+});
+
+describe('migrateLegacyOptions', () => {
+  it('should return all defaults when no options are provided', () => {
+    const { agentSessionOptions: result } = migrateLegacyOptions({});
+
+    expect(result.turnHandling).toEqual({
+      turnDetection: defaultTurnHandlingOptions.turnDetection,
+      endpointing: defaultEndpointingOptions,
+      interruption: defaultInterruptionOptions,
+    });
+    expect(result.maxToolSteps).toBe(defaultAgentSessionOptions.maxToolSteps);
+    expect(result.preemptiveGeneration).toBe(defaultAgentSessionOptions.preemptiveGeneration);
+    expect(result.userAwayTimeout).toBe(defaultAgentSessionOptions.userAwayTimeout);
+  });
+
+  it('should migrate legacy flat fields into nested turnHandling config', () => {
+    const { agentSessionOptions: result } = migrateLegacyOptions({
+      voiceOptions: {
+        minInterruptionDuration: 1000,
+        minInterruptionWords: 3,
+        discardAudioIfUninterruptible: false,
+        minEndpointingDelay: 800,
+        maxEndpointingDelay: 5000,
+      },
+    });
+
+    expect(result.turnHandling.interruption!.minDuration).toBe(1000);
+    expect(result.turnHandling.interruption!.minWords).toBe(3);
+    expect(result.turnHandling.interruption!.discardAudioIfUninterruptible).toBe(false);
+    expect(result.turnHandling.endpointing!.minDelay).toBe(800);
+    expect(result.turnHandling.endpointing!.maxDelay).toBe(5000);
+  });
+
+  it('should set interruption.enabled to false when allowInterruptions is false', () => {
+    const { agentSessionOptions: result } = migrateLegacyOptions({
+      voiceOptions: { allowInterruptions: false },
+    });
+
+    expect(result.turnHandling.interruption!.enabled).toBe(false);
+  });
+
+  it('should give top-level fields precedence over voiceOptions', () => {
+    const { agentSessionOptions: result } = migrateLegacyOptions({
+      voiceOptions: {
+        minInterruptionDuration: 1000,
+        maxEndpointingDelay: 5000,
+        maxToolSteps: 10,
+      },
+      turnHandling: {
+        interruption: {
+          minDuration: 2000,
+        },
+        endpointing: {
+          maxDelay: 8000,
+        },
+      },
+      maxToolSteps: 5,
+    });
+
+    expect(result.turnHandling.interruption!.minDuration).toBe(2000);
+    expect(result.turnHandling.endpointing!.maxDelay).toBe(8000);
+    expect(result.maxToolSteps).toBe(5);
+  });
+
+  it('should preserve top-level turnDetection in the result', () => {
+    const { agentSessionOptions: result } = migrateLegacyOptions({
+      turnDetection: 'vad',
+    });
+
+    expect(result.turnHandling.turnDetection).toBe('vad');
+  });
+});
+
+describe('migrateTurnHandling', () => {
+  it('should return empty partial when no deprecated Agent fields are given', () => {
+    const result = migrateTurnHandling({});
+    expect(result).toEqual({});
+  });
+
+  it('should set interruption.enabled to false when allowInterruptions is false', () => {
+    const result = migrateTurnHandling({ allowInterruptions: false });
+    expect(result.interruption).toEqual({ enabled: false });
+    expect(result.endpointing).toBeUndefined();
+    expect(result.turnDetection).toBeUndefined();
+  });
+
+  it('should not set interruption when allowInterruptions is true or undefined', () => {
+    expect(migrateTurnHandling({ allowInterruptions: true })).toEqual({});
+    expect(migrateTurnHandling({ allowInterruptions: undefined })).toEqual({});
+  });
+
+  it('should map minEndpointingDelay to endpointing.minDelay', () => {
+    const result = migrateTurnHandling({ minEndpointingDelay: 800 });
+    expect(result.endpointing).toEqual({ minDelay: 800 });
+  });
+
+  it('should map maxEndpointingDelay to endpointing.maxDelay', () => {
+    const result = migrateTurnHandling({ maxEndpointingDelay: 5000 });
+    expect(result.endpointing).toEqual({ maxDelay: 5000 });
+  });
+
+  it('should pass through turnDetection', () => {
+    const result = migrateTurnHandling({ turnDetection: 'vad' });
+    expect(result.turnDetection).toBe('vad');
+  });
+
+  it('should combine all deprecated Agent fields', () => {
+    const result = migrateTurnHandling({
+      turnDetection: 'stt',
+      allowInterruptions: false,
+      minEndpointingDelay: 400,
+      maxEndpointingDelay: 3000,
+    });
+    expect(result.turnDetection).toBe('stt');
+    expect(result.interruption).toEqual({ enabled: false });
+    expect(result.endpointing).toEqual({ minDelay: 400, maxDelay: 3000 });
+  });
+
+  it('should ignore deprecated Agent fields when explicit turnHandling is provided', () => {
+    const turnHandling = {
+      endpointing: { minDelay: 999, maxDelay: 4000 },
+      interruption: { enabled: true },
+      turnDetection: 'vad' as const,
+    };
+    const result = migrateTurnHandling({
+      turnHandling,
+      turnDetection: 'stt',
+      allowInterruptions: false,
+      minEndpointingDelay: 100,
+      maxEndpointingDelay: 200,
+    });
+    expect(result).toEqual(turnHandling);
+  });
+});

package/src/voice/turn_config/utils.ts

@@ -0,0 +1,167 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../../log.js';
+import {
+  type AgentSessionOptions,
+  type InternalSessionOptions,
+  type TurnDetectionMode,
+  type VoiceOptions,
+} from '../agent_session.js';
+import { defaultEndpointingOptions } from './endpointing.js';
+import { defaultInterruptionOptions } from './interruption.js';
+import { type TurnHandlingOptions, defaultTurnHandlingOptions } from './turn_handling.js';
+
+const defaultSessionOptions = {
+  maxToolSteps: 3,
+  preemptiveGeneration: true,
+  userAwayTimeout: 15.0,
+  aecWarmupDuration: 3000,
+  turnHandling: {},
+  useTtsAlignedTranscript: true,
+} as const satisfies AgentSessionOptions;
+
+const defaultLegacyVoiceOptions: VoiceOptions = {
+  minEndpointingDelay: defaultTurnHandlingOptions.endpointing.minDelay,
+  maxEndpointingDelay: defaultTurnHandlingOptions.endpointing.maxDelay,
+  maxToolSteps: defaultSessionOptions.maxToolSteps,
+  preemptiveGeneration: defaultSessionOptions.preemptiveGeneration,
+};
+
+export function migrateLegacyOptions<UserData>(legacyOptions: AgentSessionOptions<UserData>): {
+  agentSessionOptions: InternalSessionOptions<UserData>;
+  legacyVoiceOptions: VoiceOptions;
+} {
+  const logger = log();
+  const {
+    voiceOptions,
+    turnDetection,
+    stt,
+    vad,
+    llm,
+    tts,
+    userData,
+    connOptions,
+    ...sessionOptions
+  } = legacyOptions;
+
+  if (voiceOptions !== undefined) {
+    logger.warn(
+      'voiceOptions is deprecated, use top-level SessionOptions fields on AgentSessionOptions instead',
+    );
+  }
+
+  const turnHandling: TurnHandlingOptions = {
+    interruption: {
+      discardAudioIfUninterruptible: voiceOptions?.discardAudioIfUninterruptible,
+      minDuration: voiceOptions?.minInterruptionDuration,
+      minWords: voiceOptions?.minInterruptionWords,
+      ...sessionOptions.turnHandling?.interruption,
+    },
+    endpointing: {
+      minDelay: voiceOptions?.minEndpointingDelay,
+      maxDelay: voiceOptions?.maxEndpointingDelay,
+      ...sessionOptions.turnHandling?.endpointing,
+    },
+
+    turnDetection: sessionOptions?.turnHandling?.turnDetection ?? turnDetection,
+  } as const;
+
+  if (
+    voiceOptions?.allowInterruptions === false &&
+    turnHandling.interruption.enabled === undefined
+  ) {
+    turnHandling.interruption.enabled = false;
+  }
+
+  const migratedVoiceOptions: AgentSessionOptions<UserData> = {};
+
+  if (voiceOptions?.maxToolSteps !== undefined) {
+    migratedVoiceOptions.maxToolSteps = voiceOptions.maxToolSteps;
+  }
+  if (voiceOptions?.preemptiveGeneration !== undefined) {
+    migratedVoiceOptions.preemptiveGeneration = voiceOptions.preemptiveGeneration;
+  }
+  if (voiceOptions?.userAwayTimeout !== undefined) {
+    migratedVoiceOptions.userAwayTimeout = voiceOptions.userAwayTimeout;
+  }
+
+  const legacyVoiceOptions = { ...defaultLegacyVoiceOptions, ...voiceOptions };
+
+  const agentSessionOptions = {
+    stt,
+    vad,
+    llm,
+    tts,
+    userData,
+    connOptions,
+    ...defaultSessionOptions,
+    ...migratedVoiceOptions,
+    ...sessionOptions,
+    turnHandling: mergeWithDefaults(turnHandling),
+    // repopulate the deprecated voice options with migrated options for backwards compatibility
+    voiceOptions: legacyVoiceOptions,
+  };
+
+  return { agentSessionOptions, legacyVoiceOptions };
+}
+
+/** Remove keys whose value is `undefined` so they don't shadow defaults when spread. */
+export function stripUndefined<T extends object>(obj: T): Partial<T> {
+  return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined)) as Partial<T>;
+}
+
+export function mergeWithDefaults(config: TurnHandlingOptions) {
+  return {
+    turnDetection: config.turnDetection ?? defaultTurnHandlingOptions.turnDetection,
+    endpointing: { ...defaultEndpointingOptions, ...stripUndefined(config.endpointing) },
+    interruption: { ...defaultInterruptionOptions, ...stripUndefined(config.interruption) },
+  } as const;
+}
+
+/**
+ * Build a partial {@link TurnHandlingOptions} from deprecated Agent constructor fields.
+ * Mirrors the Python Agent compatibility path, but keeps the JS API surface explicit.
+ */
+export function migrateTurnHandling(opts: {
+  turnDetection?: TurnDetectionMode;
+  allowInterruptions?: boolean;
+  minEndpointingDelay?: number;
+  maxEndpointingDelay?: number;
+  turnHandling?: TurnHandlingOptions;
+}): Partial<TurnHandlingOptions> {
+  if (opts.turnHandling !== undefined) {
+    return opts.turnHandling;
+  }
+
+  const migrated: Partial<TurnHandlingOptions> = {};
+
+  const endpointing: Partial<TurnHandlingOptions['endpointing']> = {};
+  if (opts.minEndpointingDelay !== undefined) {
+    endpointing.minDelay = opts.minEndpointingDelay;
+  }
+  if (opts.maxEndpointingDelay !== undefined) {
+    endpointing.maxDelay = opts.maxEndpointingDelay;
+  }
+  if (Object.keys(endpointing).length > 0) {
+    migrated.endpointing = endpointing;
+  }
+
+  const interruption: Partial<TurnHandlingOptions['interruption']> = {};
+  if (opts.allowInterruptions === false) {
+    interruption.enabled = false;
+  }
+  if (Object.keys(interruption).length > 0) {
+    migrated.interruption = interruption;
+  }
+
+  if (opts.turnDetection !== undefined) {
+    migrated.turnDetection = opts.turnDetection;
+  }
+
+  return {
+    ...(migrated.endpointing ? { endpointing: migrated.endpointing } : {}),
+    ...(migrated.interruption ? { interruption: migrated.interruption } : {}),
+    ...(migrated.turnDetection !== undefined ? { turnDetection: migrated.turnDetection } : {}),
+  };
+}

package/src/voice/utils.ts

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Participant, ParticipantKind } from '@livekit/rtc-node';
+import type { Span } from '@opentelemetry/api';
+import { traceTypes } from '../telemetry/index.js';
+
+export function setParticipantSpanAttributes(
+  span: Span,
+  participant: Pick<Participant, 'sid' | 'identity' | 'kind'>,
+): void {
+  if (participant.sid) {
+    span.setAttribute(traceTypes.ATTR_PARTICIPANT_ID, participant.sid);
+  }
+  span.setAttribute(traceTypes.ATTR_PARTICIPANT_IDENTITY, participant.identity);
+  span.setAttribute(traceTypes.ATTR_PARTICIPANT_KIND, participantKindName(participant.kind));
+}
+
+function participantKindName(kind: ParticipantKind): string {
+  const names: Record<number, string> = {
+    0: 'STANDARD',
+    1: 'INGRESS',
+    2: 'EGRESS',
+    3: 'SIP',
+    4: 'AGENT',
+    5: 'CONNECTOR',
+  };
+  return names[kind as number] ?? String(kind);
+}