@livekit/agents 1.1.0 → 1.2.0

package/src/inference/interruption/errors.ts

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Error thrown during interruption detection.
+ */
+export class InterruptionDetectionError extends Error {
+  readonly type = 'interruption_detection_error' as const;
+
+  readonly timestamp: number;
+  readonly label: string;
+  readonly recoverable: boolean;
+
+  constructor(message: string, timestamp: number, label: string, recoverable: boolean) {
+    super(message);
+    this.name = 'InterruptionDetectionError';
+    this.timestamp = timestamp;
+    this.label = label;
+    this.recoverable = recoverable;
+  }
+
+  toString(): string {
+    return `${this.name}: ${this.message} (label=${this.label}, timestamp=${this.timestamp}, recoverable=${this.recoverable})`;
+  }
+}

package/src/inference/interruption/http_transport.ts

@@ -0,0 +1,206 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { FetchError, ofetch } from 'ofetch';
+import { TransformStream } from 'stream/web';
+import { z } from 'zod';
+import { APIConnectionError, APIError, APIStatusError, isAPIError } from '../../_exceptions.js';
+import { log } from '../../log.js';
+import { createAccessToken } from '../utils.js';
+import { InterruptionCacheEntry } from './interruption_cache_entry.js';
+import type { OverlappingSpeechEvent } from './types.js';
+import type { BoundedCache } from './utils.js';
+
+export interface PostOptions {
+  baseUrl: string;
+  token: string;
+  signal?: AbortSignal;
+  timeout?: number;
+  maxRetries?: number;
+}
+
+export interface PredictOptions {
+  threshold: number;
+  minFrames: number;
+}
+
+export const predictEndpointResponseSchema = z.object({
+  created_at: z.number(),
+  is_bargein: z.boolean(),
+  probabilities: z.array(z.number()),
+});
+
+export type PredictEndpointResponse = z.infer<typeof predictEndpointResponseSchema>;
+
+export interface PredictResponse {
+  createdAt: number;
+  isBargein: boolean;
+  probabilities: number[];
+  predictionDurationInS: number;
+}
+
+export async function predictHTTP(
+  data: Int16Array,
+  predictOptions: PredictOptions,
+  options: PostOptions,
+): Promise<PredictResponse> {
+  const createdAt = performance.now();
+  const url = new URL(`/bargein`, options.baseUrl);
+  url.searchParams.append('threshold', predictOptions.threshold.toString());
+  url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());
+  url.searchParams.append('created_at', createdAt.toFixed());
+
+  try {
+    const response = await ofetch(url.toString(), {
+      retry: 0,
+      headers: {
+        'Content-Type': 'application/octet-stream',
+        Authorization: `Bearer ${options.token}`,
+      },
+      signal: options.signal,
+      timeout: options.timeout,
+      method: 'POST',
+      body: data,
+    });
+    const { created_at, is_bargein, probabilities } = predictEndpointResponseSchema.parse(response);
+
+    return {
+      createdAt: created_at,
+      isBargein: is_bargein,
+      probabilities,
+      predictionDurationInS: (performance.now() - createdAt) / 1000,
+    };
+  } catch (err) {
+    if (isAPIError(err)) throw err;
+    if (err instanceof FetchError) {
+      if (err.statusCode) {
+        throw new APIStatusError({
+          message: `error during interruption prediction: ${err.message}`,
+          options: { statusCode: err.statusCode, body: err.data },
+        });
+      }
+      if (
+        err.cause instanceof Error &&
+        (err.cause.name === 'TimeoutError' || err.cause.name === 'AbortError')
+      ) {
+        throw new APIStatusError({
+          message: `interruption inference timeout: ${err.message}`,
+          options: { statusCode: 408, retryable: false },
+        });
+      }
+      throw new APIConnectionError({
+        message: `interruption inference connection error: ${err.message}`,
+      });
+    }
+    throw new APIError(`error during interruption prediction: ${err}`);
+  }
+}
+
+export interface HttpTransportOptions {
+  baseUrl: string;
+  apiKey: string;
+  apiSecret: string;
+  threshold: number;
+  minFrames: number;
+  timeout: number;
+  maxRetries?: number;
+}
+
+export interface HttpTransportState {
+  overlapSpeechStarted: boolean;
+  overlapSpeechStartedAt: number | undefined;
+  cache: BoundedCache<number, InterruptionCacheEntry>;
+}
+
+/**
+ * Creates an HTTP transport TransformStream for interruption detection.
+ *
+ * This transport receives Int16Array audio slices and outputs InterruptionEvents.
+ * Each audio slice triggers an HTTP POST request.
+ *
+ * @param options - Transport options object. This is read on each request, so mutations
+ *                  to threshold/minFrames will be picked up dynamically.
+ */
+export function createHttpTransport(
+  options: HttpTransportOptions,
+  getState: () => HttpTransportState,
+  setState: (partial: Partial<HttpTransportState>) => void,
+  updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,
+  getAndResetNumRequests?: () => number,
+): TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent> {
+  const logger = log();
+
+  return new TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>(
+    {
+      async transform(chunk, controller) {
+        if (!(chunk instanceof Int16Array)) {
+          controller.enqueue(chunk);
+          return;
+        }
+
+        const state = getState();
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (overlapSpeechStartedAt === undefined || !state.overlapSpeechStarted) return;
+
+        try {
+          const resp = await predictHTTP(
+            chunk,
+            { threshold: options.threshold, minFrames: options.minFrames },
+            {
+              baseUrl: options.baseUrl,
+              timeout: options.timeout,
+              maxRetries: options.maxRetries,
+              token: await createAccessToken(options.apiKey, options.apiSecret),
+            },
+          );
+
+          const { createdAt, isBargein, probabilities, predictionDurationInS } = resp;
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              probabilities,
+              isInterruption: isBargein,
+              speechInput: chunk,
+              totalDurationInS: (performance.now() - createdAt) / 1000,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,
+              predictionDurationInS,
+            },
+          );
+
+          if (state.overlapSpeechStarted && entry.isInterruption) {
+            if (updateUserSpeakingSpan) {
+              updateUserSpeakingSpan(entry);
+            }
+            const event: OverlappingSpeechEvent = {
+              type: 'overlapping_speech',
+              detectedAt: Date.now(),
+              overlapStartedAt: overlapSpeechStartedAt,
+              isInterruption: entry.isInterruption,
+              speechInput: entry.speechInput,
+              probabilities: entry.probabilities,
+              totalDurationInS: entry.totalDurationInS,
+              predictionDurationInS: entry.predictionDurationInS,
+              detectionDelayInS: entry.detectionDelayInS,
+              probability: entry.probability,
+              numRequests: getAndResetNumRequests?.() ?? 0,
+            };
+            logger.debug(
+              {
+                detectionDelayInS: entry.detectionDelayInS,
+                totalDurationInS: entry.totalDurationInS,
+              },
+              'interruption detected',
+            );
+            setState({ overlapSpeechStarted: false });
+            controller.enqueue(event);
+          }
+        } catch (err) {
+          controller.error(err);
+        }
+      },
+    },
+    { highWaterMark: 2 },
+    { highWaterMark: 2 },
+  );
+}

package/src/inference/interruption/interruption_cache_entry.ts

@@ -0,0 +1,50 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { estimateProbability } from './utils.js';
+
+/**
+ * Typed cache entry for interruption inference results.
+ * Mutable to support setOrUpdate pattern from Python's _BoundedCache.
+ */
+export class InterruptionCacheEntry {
+  createdAt: number;
+  requestStartedAt?: number;
+  totalDurationInS: number;
+  predictionDurationInS: number;
+  detectionDelayInS: number;
+  speechInput?: Int16Array;
+  probabilities?: number[];
+  isInterruption?: boolean;
+
+  constructor(params: {
+    createdAt: number;
+    requestStartedAt?: number;
+    speechInput?: Int16Array;
+    totalDurationInS?: number;
+    predictionDurationInS?: number;
+    detectionDelayInS?: number;
+    probabilities?: number[];
+    isInterruption?: boolean;
+  }) {
+    this.createdAt = params.createdAt;
+    this.requestStartedAt = params.requestStartedAt;
+    this.totalDurationInS = params.totalDurationInS ?? 0;
+    this.predictionDurationInS = params.predictionDurationInS ?? 0;
+    this.detectionDelayInS = params.detectionDelayInS ?? 0;
+    this.speechInput = params.speechInput;
+    this.probabilities = params.probabilities;
+    this.isInterruption = params.isInterruption;
+  }
+
+  /**
+   * The conservative estimated probability of the interruption event.
+   */
+  get probability(): number {
+    return this.probabilities ? estimateProbability(this.probabilities) : 0;
+  }
+
+  static default(): InterruptionCacheEntry {
+    return new InterruptionCacheEntry({ createdAt: 0 });
+  }
+}

package/src/inference/interruption/interruption_detector.ts

@@ -0,0 +1,204 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { TypedEventEmitter } from '@livekit/typed-emitter';
+import EventEmitter from 'events';
+import { log } from '../../log.js';
+import type { InterruptionMetrics } from '../../metrics/base.js';
+import { DEFAULT_INFERENCE_URL, STAGING_INFERENCE_URL, getDefaultInferenceUrl } from '../utils.js';
+import { FRAMES_PER_SECOND, SAMPLE_RATE, interruptionOptionDefaults } from './defaults.js';
+import { InterruptionDetectionError } from './errors.js';
+import { InterruptionStreamBase } from './interruption_stream.js';
+import type { InterruptionOptions, OverlappingSpeechEvent } from './types.js';
+
+type InterruptionCallbacks = {
+  overlapping_speech: (event: OverlappingSpeechEvent) => void;
+  metrics_collected: (metrics: InterruptionMetrics) => void;
+  error: (error: InterruptionDetectionError) => void;
+};
+
+export type AdaptiveInterruptionDetectorOptions = Omit<Partial<InterruptionOptions>, 'useProxy'>;
+
+export class AdaptiveInterruptionDetector extends (EventEmitter as new () => TypedEventEmitter<InterruptionCallbacks>) {
+  options: InterruptionOptions;
+  private readonly _label: string;
+  private logger = log();
+  // Use Set instead of WeakSet to allow iteration for propagating option updates
+  private streams: Set<InterruptionStreamBase> = new Set();
+
+  constructor(options: AdaptiveInterruptionDetectorOptions = {}) {
+    super();
+
+    const {
+      maxAudioDurationInS,
+      baseUrl,
+      apiKey,
+      apiSecret,
+      audioPrefixDurationInS,
+      threshold,
+      detectionIntervalInS,
+      inferenceTimeout,
+      minInterruptionDurationInS,
+    } = { ...interruptionOptionDefaults, ...options };
+
+    if (maxAudioDurationInS > 3.0) {
+      throw new RangeError('maxAudioDurationInS must be less than or equal to 3.0 seconds');
+    }
+
+    const lkBaseUrl = baseUrl ?? process.env.LIVEKIT_REMOTE_EOT_URL ?? getDefaultInferenceUrl();
+    let lkApiKey = apiKey ?? '';
+    let lkApiSecret = apiSecret ?? '';
+    let useProxy: boolean;
+
+    // Use LiveKit credentials if using the inference service (production or staging)
+    const isInferenceUrl =
+      lkBaseUrl === DEFAULT_INFERENCE_URL || lkBaseUrl === STAGING_INFERENCE_URL;
+    if (isInferenceUrl) {
+      lkApiKey =
+        apiKey ?? process.env.LIVEKIT_INFERENCE_API_KEY ?? process.env.LIVEKIT_API_KEY ?? '';
+      if (!lkApiKey) {
+        throw new TypeError(
+          'apiKey is required, either as argument or set LIVEKIT_API_KEY environmental variable',
+        );
+      }
+
+      lkApiSecret =
+        apiSecret ??
+        process.env.LIVEKIT_INFERENCE_API_SECRET ??
+        process.env.LIVEKIT_API_SECRET ??
+        '';
+      if (!lkApiSecret) {
+        throw new TypeError(
+          'apiSecret is required, either as argument or set LIVEKIT_API_SECRET environmental variable',
+        );
+      }
+      useProxy = true;
+    } else {
+      useProxy = false;
+    }
+    const transport = useProxy ? 'websocket' : 'http';
+    this.logger.debug(
+      {
+        baseUrl: lkBaseUrl,
+        useProxy,
+        transport,
+      },
+      '=== Resolved interruption detector transport configuration',
+    );
+
+    this.options = {
+      sampleRate: SAMPLE_RATE,
+      threshold,
+      minFrames: Math.ceil(minInterruptionDurationInS * FRAMES_PER_SECOND),
+      maxAudioDurationInS,
+      audioPrefixDurationInS,
+      detectionIntervalInS,
+      inferenceTimeout,
+      baseUrl: lkBaseUrl,
+      apiKey: lkApiKey,
+      apiSecret: lkApiSecret,
+      useProxy,
+      minInterruptionDurationInS,
+    };
+
+    this._label = `${this.constructor.name}`;
+
+    this.logger.debug(
+      {
+        baseUrl: this.options.baseUrl,
+        detectionIntervalInS: this.options.detectionIntervalInS,
+        audioPrefixDurationInS: this.options.audioPrefixDurationInS,
+        maxAudioDurationInS: this.options.maxAudioDurationInS,
+        minFrames: this.options.minFrames,
+        threshold: this.options.threshold,
+        inferenceTimeout: this.options.inferenceTimeout,
+        useProxy: this.options.useProxy,
+        transport,
+      },
+      '=== Adaptive interruption detector initialized',
+    );
+  }
+
+  /**
+   * The model identifier for this detector.
+   */
+  get model(): string {
+    return 'adaptive interruption';
+  }
+
+  /**
+   * The provider identifier for this detector.
+   */
+  get provider(): string {
+    return 'livekit';
+  }
+
+  /**
+   * The label for this detector instance.
+   */
+  get label(): string {
+    return this._label;
+  }
+
+  /**
+   * The sample rate used for audio processing.
+   */
+  get sampleRate(): number {
+    return this.options.sampleRate;
+  }
+
+  /**
+   * Emit an error event from the detector.
+   */
+  emitError(error: InterruptionDetectionError): void {
+    this.emit('error', error);
+  }
+
+  /**
+   * Creates a new InterruptionStreamBase for internal use.
+   * The stream can receive audio frames and sentinels via pushFrame().
+   * Use this when you need direct access to the stream for pushing frames.
+   */
+  createStream(): InterruptionStreamBase {
+    try {
+      const streamBase = new InterruptionStreamBase(this, {});
+      this.streams.add(streamBase);
+      return streamBase;
+    } catch (e) {
+      const cause = e instanceof Error ? e : new Error(String(e));
+      this.emitError(new InterruptionDetectionError(cause.message, Date.now(), this._label, false));
+      throw e;
+    }
+  }
+
+  /**
+   * Remove a stream from tracking (called when stream is closed).
+   */
+  removeStream(stream: InterruptionStreamBase): void {
+    this.streams.delete(stream);
+  }
+
+  /**
+   * Update options for the detector and propagate to all active streams.
+   * For WebSocket streams, this triggers a reconnection with new settings.
+   */
+  async updateOptions(options: {
+    threshold?: number;
+    minInterruptionDurationInS?: number;
+  }): Promise<void> {
+    if (options.threshold !== undefined) {
+      this.options.threshold = options.threshold;
+    }
+    if (options.minInterruptionDurationInS !== undefined) {
+      this.options.minInterruptionDurationInS = options.minInterruptionDurationInS;
+      this.options.minFrames = Math.ceil(options.minInterruptionDurationInS * FRAMES_PER_SECOND);
+    }
+
+    // Propagate option updates to all active streams (matching Python behavior)
+    const updatePromises: Promise<void>[] = [];
+    for (const stream of this.streams) {
+      updatePromises.push(stream.updateOptions(options));
+    }
+    await Promise.all(updatePromises);
+  }
+}