@livekit/agents 1.0.47 → 1.1.0-dev.0

package/src/voice/testing/fake_llm.ts

@@ -0,0 +1,138 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext } from '../../llm/chat_context.js';
+import { FunctionCall } from '../../llm/chat_context.js';
+import { LLMStream as BaseLLMStream, LLM, type LLMStream } from '../../llm/llm.js';
+import type { ToolChoice, ToolContext } from '../../llm/tool_context.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../../types.js';
+import { delay } from '../../utils.js';
+
+export interface FakeLLMResponse {
+  input: string;
+  type?: 'llm';
+  content?: string;
+  ttft?: number;
+  duration?: number;
+  toolCalls?: Array<{ name: string; args: Record<string, unknown> }>;
+}
+
+export class FakeLLM extends LLM {
+  private readonly responseMap = new Map<string, FakeLLMResponse>();
+
+  constructor(responses: FakeLLMResponse[] = []) {
+    super();
+    for (const response of responses) {
+      this.responseMap.set(response.input, {
+        type: 'llm',
+        ttft: 0,
+        duration: 0,
+        ...response,
+      });
+    }
+  }
+
+  label(): string {
+    return 'fake-llm';
+  }
+
+  chat({
+    chatCtx,
+    toolCtx,
+    connOptions = DEFAULT_API_CONNECT_OPTIONS,
+  }: {
+    chatCtx: ChatContext;
+    toolCtx?: ToolContext;
+    connOptions?: APIConnectOptions;
+    parallelToolCalls?: boolean;
+    toolChoice?: ToolChoice;
+    extraKwargs?: Record<string, unknown>;
+  }): LLMStream {
+    return new FakeLLMStream(this, {
+      chatCtx,
+      toolCtx,
+      connOptions,
+    });
+  }
+
+  lookup(input: string): FakeLLMResponse | undefined {
+    return this.responseMap.get(input);
+  }
+}
+
+class FakeLLMStream extends BaseLLMStream {
+  private readonly fake: FakeLLM;
+
+  constructor(
+    fake: FakeLLM,
+    params: { chatCtx: ChatContext; toolCtx?: ToolContext; connOptions: APIConnectOptions },
+  ) {
+    super(fake, params);
+    this.fake = fake;
+  }
+
+  protected async run(): Promise<void> {
+    const input = this.getInputText();
+    const decision = this.fake.lookup(input);
+    if (!decision) {
+      return;
+    }
+
+    const startedAt = Date.now();
+    if ((decision.ttft ?? 0) > 0) {
+      await delay(decision.ttft!);
+    }
+
+    const content = decision.content ?? '';
+    const chunkSize = 3;
+    for (let i = 0; i < content.length; i += chunkSize) {
+      this.queue.put({
+        id: 'fake',
+        delta: { role: 'assistant', content: content.slice(i, i + chunkSize) },
+      });
+    }
+
+    if (decision.toolCalls && decision.toolCalls.length > 0) {
+      const calls = decision.toolCalls.map((tc, index) =>
+        FunctionCall.create({
+          callId: `fake_call_${index}`,
+          name: tc.name,
+          args: JSON.stringify(tc.args),
+        }),
+      );
+      this.queue.put({
+        id: 'fake',
+        delta: { role: 'assistant', toolCalls: calls },
+      });
+    }
+
+    const elapsed = Date.now() - startedAt;
+    const waitMs = Math.max(0, (decision.duration ?? 0) - elapsed);
+    if (waitMs > 0) {
+      await delay(waitMs);
+    }
+  }
+
+  private getInputText(): string {
+    const items = this.chatCtx.items;
+    if (items.length === 0) {
+      throw new Error('No input text found');
+    }
+
+    for (const item of items) {
+      if (item.type === 'message' && item.role === 'system') {
+        const text = item.textContent ?? '';
+        const lines = text.split('\n');
+        const tail = lines[lines.length - 1] ?? '';
+        if (lines.length > 1 && tail.startsWith('instructions:')) {
+          return tail;
+        }
+      }
+    }
+
+    const last = items[items.length - 1]!;
+    if (last.type === 'message' && last.role === 'user') return last.textContent ?? '';
+    if (last.type === 'function_call_output') return last.output;
+    throw new Error('No input text found');
+  }
+}

package/src/voice/testing/index.ts

@@ -48,3 +48,5 @@ export {
   type MessageAssertOptions,
   type RunEvent,
 } from './types.js';
+
+export { FakeLLM, type FakeLLMResponse } from './fake_llm.js';

package/src/voice/turn_config/endpointing.ts

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Configuration for endpointing, which determines when the user's turn is complete.
+ */
+export interface EndpointingOptions {
+  /**
+   * Endpointing mode. `"fixed"` uses a fixed delay, `"dynamic"` adjusts delay based on
+   * end-of-utterance prediction.
+   * @defaultValue "fixed"
+   */
+  mode: 'fixed' | 'dynamic';
+  /**
+   * Minimum time in milliseconds since the last detected speech before the agent declares the user's
+   * turn complete. In VAD mode this effectively behaves like `max(VAD silence, minDelay)`;
+   * in STT mode it is applied after the STT end-of-speech signal, so it can be additive with
+   * the STT provider's endpointing delay.
+   * @defaultValue 500
+   */
+  minDelay: number;
+  /**
+   * Maximum time in milliseconds the agent will wait before terminating the turn.
+   * @defaultValue 3000
+   */
+  maxDelay: number;
+}
+
+export const defaultEndpointingOptions = {
+  mode: 'fixed',
+  minDelay: 500,
+  maxDelay: 3000,
+} as const satisfies EndpointingOptions;

package/src/voice/turn_config/interruption.ts

@@ -0,0 +1,56 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Configuration for interruption handling.
+ */
+export interface InterruptionOptions {
+  /**
+   * Whether interruptions are enabled.
+   * @defaultValue true
+   */
+  enabled: boolean;
+  /**
+   * Interruption handling strategy. `"adaptive"` for ML-based detection, `"vad"` for simple
+   * voice-activity detection. `undefined` means auto-detect.
+   * @defaultValue undefined
+   */
+  mode: 'adaptive' | 'vad' | false | undefined;
+  /**
+   * When `true`, buffered audio is dropped while the agent is speaking and cannot be interrupted.
+   * @defaultValue true
+   */
+  discardAudioIfUninterruptible: boolean;
+  /**
+   * Minimum speech length in milliseconds to register as an interruption.
+   * @defaultValue 500
+   */
+  minDuration: number;
+  /**
+   * Minimum number of words to consider an interruption, only used if STT is enabled.
+   * @defaultValue 0
+   */
+  minWords: number;
+  /**
+   * If set, emit an `agentFalseInterruption` event after this amount of time if the user is
+   * silent and no user transcript is detected after the interruption. Set to `undefined` to
+   * disable. The value is in milliseconds.
+   * @defaultValue 2000
+   */
+  falseInterruptionTimeout: number;
+  /**
+   * Whether to resume the false interruption after the `falseInterruptionTimeout`.
+   * @defaultValue true
+   */
+  resumeFalseInterruption: boolean;
+}
+
+export const defaultInterruptionOptions = {
+  enabled: true,
+  mode: undefined,
+  discardAudioIfUninterruptible: true,
+  minDuration: 500,
+  minWords: 0,
+  falseInterruptionTimeout: 2000,
+  resumeFalseInterruption: true,
+} as const satisfies InterruptionOptions;

package/src/voice/turn_config/turn_handling.ts

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { TurnDetectionMode } from '../agent_session.js';
+import { type EndpointingOptions, defaultEndpointingOptions } from './endpointing.js';
+import { type InterruptionOptions, defaultInterruptionOptions } from './interruption.js';
+
+/**
+ * Configuration for the turn handling system. Used to configure the turn taking behavior of the
+ * session.
+ */
+export interface TurnHandlingOptions {
+  /**
+   * Strategy for deciding when the user has finished speaking.
+   *
+   * - `"stt"` – rely on speech-to-text end-of-utterance cues
+   * - `"vad"` – rely on Voice Activity Detection start/stop cues
+   * - `"realtime_llm"` – use server-side detection from a realtime LLM
+   * - `"manual"` – caller controls turn boundaries explicitly
+   *
+   * If not set, the session chooses the best available mode in priority order
+   * `realtime_llm → vad → stt → manual`; it automatically falls back if the necessary model
+   * is missing.
+   */
+  turnDetection: TurnDetectionMode | undefined;
+  /**
+   * Configuration for endpointing.
+   */
+  endpointing: Partial<EndpointingOptions>;
+  /**
+   * Configuration for interruption handling.
+   */
+  interruption: Partial<InterruptionOptions>;
+}
+
+export interface InternalTurnHandlingOptions extends TurnHandlingOptions {
+  endpointing: EndpointingOptions;
+  interruption: InterruptionOptions;
+}
+
+export const defaultTurnHandlingOptions: InternalTurnHandlingOptions = {
+  turnDetection: undefined,
+  interruption: defaultInterruptionOptions,
+  endpointing: defaultEndpointingOptions,
+};

package/src/voice/turn_config/utils.test.ts

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { beforeAll, describe, expect, it } from 'vitest';
+import { initializeLogger } from '../../log.js';
+import { defaultEndpointingOptions } from './endpointing.js';
+import { defaultInterruptionOptions } from './interruption.js';
+import { defaultTurnHandlingOptions } from './turn_handling.js';
+import { migrateLegacyOptions } from './utils.js';
+
+beforeAll(() => {
+  initializeLogger({ pretty: true, level: 'info' });
+});
+
+describe('migrateLegacyOptions', () => {
+  it('should return all defaults when no options are provided', () => {
+    const result = migrateLegacyOptions({});
+
+    expect(result.options.turnHandling).toEqual({
+      turnDetection: defaultTurnHandlingOptions.turnDetection,
+      endpointing: defaultEndpointingOptions,
+      interruption: defaultInterruptionOptions,
+    });
+    expect(result.options.maxToolSteps).toBe(3);
+    expect(result.options.preemptiveGeneration).toBe(false);
+    expect(result.options.userAwayTimeout).toBe(15.0);
+  });
+
+  it('should migrate legacy flat fields into nested turnHandling config', () => {
+    const result = migrateLegacyOptions({
+      voiceOptions: {
+        minInterruptionDuration: 1000,
+        minInterruptionWords: 3,
+        discardAudioIfUninterruptible: false,
+        minEndpointingDelay: 800,
+        maxEndpointingDelay: 5000,
+      },
+    });
+
+    expect(result.options.turnHandling.interruption!.minDuration).toBe(1000);
+    expect(result.options.turnHandling.interruption!.minWords).toBe(3);
+    expect(result.options.turnHandling.interruption!.discardAudioIfUninterruptible).toBe(false);
+    expect(result.options.turnHandling.endpointing!.minDelay).toBe(800);
+    expect(result.options.turnHandling.endpointing!.maxDelay).toBe(5000);
+  });
+
+  it('should set interruption.enabled to false when allowInterruptions is false', () => {
+    const result = migrateLegacyOptions({
+      options: {
+        allowInterruptions: false,
+      },
+    });
+
+    expect(result.options.turnHandling.interruption!.enabled).toBe(false);
+  });
+
+  it('should give options precedence over voiceOptions when both are provided', () => {
+    const result = migrateLegacyOptions({
+      voiceOptions: {
+        minInterruptionDuration: 1000,
+        maxEndpointingDelay: 5000,
+        maxToolSteps: 10,
+      },
+      options: {
+        minInterruptionDuration: 2000,
+        maxEndpointingDelay: 8000,
+        maxToolSteps: 5,
+      },
+    });
+
+    expect(result.options.turnHandling.interruption!.minDuration).toBe(2000);
+    expect(result.options.turnHandling.endpointing!.maxDelay).toBe(8000);
+    expect(result.options.maxToolSteps).toBe(5);
+  });
+
+  it('should let explicit turnHandling override legacy flat fields', () => {
+    const result = migrateLegacyOptions({
+      options: {
+        minInterruptionDuration: 1000,
+        minEndpointingDelay: 800,
+        turnHandling: {
+          interruption: { minDuration: 3000 },
+          endpointing: { minDelay: 2000 },
+        },
+      },
+    });
+
+    expect(result.options.turnHandling.interruption!.minDuration).toBe(3000);
+    expect(result.options.turnHandling.endpointing!.minDelay).toBe(2000);
+  });
+
+  it('should preserve top-level turnDetection in the result', () => {
+    const result = migrateLegacyOptions({
+      turnDetection: 'vad',
+    });
+
+    expect(result.turnDetection).toBe('vad');
+    expect(result.options.turnHandling.turnDetection).toBe('vad');
+  });
+});

package/src/voice/turn_config/utils.ts

@@ -0,0 +1,103 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../../log.js';
+import {
+  type AgentSessionOptions,
+  type InternalSessionOptions,
+  defaultSessionOptions,
+} from '../agent_session.js';
+import { defaultEndpointingOptions } from './endpointing.js';
+import { defaultInterruptionOptions } from './interruption.js';
+import { type TurnHandlingOptions, defaultTurnHandlingOptions } from './turn_handling.js';
+
+export function migrateLegacyOptions<UserData>(
+  legacyOptions: AgentSessionOptions<UserData>,
+): AgentSessionOptions<UserData> & { options: InternalSessionOptions } {
+  const logger = log();
+  const { voiceOptions, turnDetection, options: sessionOptions, ...rest } = legacyOptions;
+
+  if (voiceOptions !== undefined && sessionOptions !== undefined) {
+    logger.warn(
+      'Both voiceOptions and options have been supplied as part of the AgentSessionOptions, voiceOptions will be merged with options taking precedence',
+    );
+  }
+
+  // Preserve turnDetection before cloning since structuredClone converts class instances to plain objects
+  const originalTurnDetection =
+    sessionOptions?.turnHandling?.turnDetection ??
+    voiceOptions?.turnHandling?.turnDetection ??
+    turnDetection;
+
+  // Exclude potentially non-cloneable turnDetection objects before structuredClone.
+  // They are restored from originalTurnDetection below.
+  const cloneableVoiceOptions = voiceOptions
+    ? {
+        ...voiceOptions,
+        turnHandling: voiceOptions.turnHandling
+          ? { ...voiceOptions.turnHandling, turnDetection: undefined }
+          : voiceOptions.turnHandling,
+      }
+    : voiceOptions;
+  const cloneableSessionOptions = sessionOptions
+    ? {
+        ...sessionOptions,
+        turnHandling: sessionOptions.turnHandling
+          ? { ...sessionOptions.turnHandling, turnDetection: undefined }
+          : sessionOptions.turnHandling,
+      }
+    : sessionOptions;
+
+  const mergedOptions = structuredClone({ ...cloneableVoiceOptions, ...cloneableSessionOptions });
+
+  const turnHandling: TurnHandlingOptions = {
+    interruption: {
+      discardAudioIfUninterruptible: mergedOptions?.discardAudioIfUninterruptible,
+      minDuration: mergedOptions?.minInterruptionDuration,
+      minWords: mergedOptions?.minInterruptionWords,
+    },
+    endpointing: {
+      minDelay: mergedOptions?.minEndpointingDelay,
+      maxDelay: mergedOptions?.maxEndpointingDelay,
+    },
+
+    ...mergedOptions.turnHandling,
+    // Restore original turnDetection after spread to preserve class instance with methods
+    // (structuredClone converts class instances to plain objects, losing prototype methods)
+    turnDetection: originalTurnDetection,
+  } as const;
+
+  if (mergedOptions?.allowInterruptions === false) {
+    turnHandling.interruption.enabled = false;
+  }
+
+  const optionsWithDefaults = {
+    ...defaultSessionOptions,
+    ...mergedOptions,
+    turnHandling: mergeWithDefaults(turnHandling),
+  };
+
+  const newAgentSessionOptions: AgentSessionOptions<UserData> & {
+    options: InternalSessionOptions;
+  } = {
+    ...rest,
+    options: optionsWithDefaults,
+    voiceOptions: optionsWithDefaults,
+    turnDetection: turnHandling.turnDetection,
+  };
+
+  return newAgentSessionOptions;
+}
+
+/** Remove keys whose value is `undefined` so they don't shadow defaults when spread. */
+export function stripUndefined<T extends object>(obj: T): Partial<T> {
+  return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined)) as Partial<T>;
+}
+
+export function mergeWithDefaults(config: TurnHandlingOptions) {
+  return {
+    turnDetection: config.turnDetection ?? defaultTurnHandlingOptions.turnDetection,
+    endpointing: { ...defaultEndpointingOptions, ...stripUndefined(config.endpointing) },
+    interruption: { ...defaultInterruptionOptions, ...stripUndefined(config.interruption) },
+  } as const;
+}