@livekit/agents 1.0.36-dev.0 → 1.0.37

package/src/inference/interruption/InterruptionStream.ts

@@ -1,397 +0,0 @@
-import { AudioFrame, AudioResampler } from '@livekit/rtc-node';
-import type { Span } from '@opentelemetry/api';
-import { traceTypes } from '../../telemetry/index.js';
-import { type ReadableStream, TransformStream, WritableStream } from 'stream/web';
-import { log } from '../../log.js';
-import { type StreamChannel, createStreamChannel } from '../../stream/stream_channel.js';
-import { createAccessToken } from '../utils.js';
-import type {
-  AdaptiveInterruptionDetector,
-  InterruptionOptions,
-} from './AdaptiveInterruptionDetector.js';
-import { apiConnectDefaults } from './defaults.js';
-import { predictHTTP } from './http_transport.js';
-import {
-  InterruptionCacheEntry,
-  type InterruptionDetectionError,
-  type InterruptionEvent,
-  InterruptionEventType,
-} from './interruption.js';
-
-export interface AgentSpeechStarted {
-  type: 'agent-speech-started';
-}
-
-export interface AgentSpeechEnded {
-  type: 'agent-speech-ended';
-}
-
-export interface OverlapSpeechStarted {
-  type: 'overlap-speech-started';
-  speechDuration: number;
-  userSpeakingSpan: Span;
-}
-
-export interface OverlapSpeechEnded {
-  type: 'overlap-speech-ended';
-}
-
-export interface Flush {
-  type: 'flush';
-}
-
-export type InterruptionSentinel =
-  | AgentSpeechStarted
-  | AgentSpeechEnded
-  | OverlapSpeechStarted
-  | OverlapSpeechEnded
-  | Flush;
-
-export class InterruptionStreamSentinel {
-  static speechStarted(): AgentSpeechStarted {
-    return { type: 'agent-speech-started' };
-  }
-
-  static speechEnded(): AgentSpeechEnded {
-    return { type: 'agent-speech-ended' };
-  }
-
-  static overlapSpeechStarted(
-    speechDuration: number,
-    userSpeakingSpan: Span,
-  ): OverlapSpeechStarted {
-    return { type: 'overlap-speech-started', speechDuration, userSpeakingSpan };
-  }
-
-  static overlapSpeechEnded(): OverlapSpeechEnded {
-    return { type: 'overlap-speech-ended' };
-  }
-
-  static flush(): Flush {
-    return { type: 'flush' };
-  }
-}
-
-export interface ApiConnectOptions {
-  maxRetries: number;
-  retryInterval: number;
-  timeout: number;
-}
-
-function updateUserSpeakingSpan(span: Span, entry: InterruptionCacheEntry) {
-  span.setAttribute(
-    traceTypes.ATTR_IS_INTERRUPTION,
-    (entry.isInterruption ?? false).toString().toLowerCase(),
-  );
-  span.setAttribute(traceTypes.ATTR_INTERRUPTION_PROBABILITY, entry.probability);
-  span.setAttribute(traceTypes.ATTR_INTERRUPTION_TOTAL_DURATION, entry.totalDuration);
-  span.setAttribute(traceTypes.ATTR_INTERRUPTION_PREDICTION_DURATION, entry.predictionDuration);
-  span.setAttribute(traceTypes.ATTR_INTERRUPTION_DETECTION_DELAY, entry.detectionDelay);
-}
-
-export class InterruptionStreamBase {
-  private inputStream: StreamChannel<InterruptionSentinel | AudioFrame, InterruptionDetectionError>;
-
-  private eventStream: StreamChannel<InterruptionEvent, InterruptionDetectionError>;
-
-  private resampler?: AudioResampler;
-
-  private userSpeakingSpan: Span | undefined;
-
-  private overlapSpeechStartedAt: number | undefined;
-
-  private options: InterruptionOptions;
-
-  private apiOptions: ApiConnectOptions;
-
-  private model: AdaptiveInterruptionDetector;
-
-  constructor(model: AdaptiveInterruptionDetector, apiOptions: Partial<ApiConnectOptions>) {
-    this.inputStream = createStreamChannel<
-      InterruptionSentinel | AudioFrame,
-      InterruptionDetectionError
-    >();
-
-    this.eventStream = createStreamChannel<InterruptionEvent, InterruptionDetectionError>();
-
-    this.model = model;
-    this.options = model.options;
-    this.apiOptions = { ...apiConnectDefaults, ...apiOptions };
-
-    this.setupTransform();
-  }
-
-  private setupTransform() {
-    let agentSpeechStarted = false;
-    let startIdx = 0;
-    let accumulatedSamples = 0;
-    let overlapSpeechStarted = false;
-    const cache = new Map<number, InterruptionCacheEntry>(); // TODO limit cache size
-    const inferenceS16Data = new Int16Array(
-      Math.ceil(this.options.maxAudioDuration * this.options.sampleRate),
-    ).fill(0);
-
-    const transformer = new TransformStream<InterruptionSentinel | AudioFrame, Int16Array>(
-      {
-        transform: (chunk, controller) => {
-          if (chunk instanceof AudioFrame) {
-            if (!agentSpeechStarted) {
-              return;
-            }
-            if (this.options.sampleRate !== chunk.sampleRate) {
-              controller.error('the sample rate of the input frames must be consistent');
-              return;
-            }
-            const result = writeToInferenceS16Data(
-              chunk,
-              startIdx,
-              inferenceS16Data,
-              this.options.maxAudioDuration,
-            );
-            startIdx = result.startIdx;
-            accumulatedSamples += result.samplesWritten;
-
-            // Send data for inference when enough samples accumulated during overlap
-            if (
-              accumulatedSamples >=
-                Math.floor(this.options.detectionInterval * this.options.sampleRate) &&
-              overlapSpeechStarted
-            ) {
-              // Send a copy of the audio data up to startIdx for inference
-              const audioSlice = inferenceS16Data.slice(0, startIdx);
-              // TODO: send to data channel - dataChan.send(audioSlice);
-              accumulatedSamples = 0;
-              controller.enqueue(audioSlice);
-            }
-          } else if (chunk.type === 'agent-speech-started') {
-            log().debug('agent speech started');
-
-            agentSpeechStarted = true;
-            overlapSpeechStarted = false;
-            accumulatedSamples = 0;
-            startIdx = 0;
-            cache.clear();
-          } else if (chunk.type === 'agent-speech-ended') {
-            log().debug('agent speech ended');
-
-            agentSpeechStarted = false;
-            overlapSpeechStarted = false;
-            accumulatedSamples = 0;
-            startIdx = 0;
-            cache.clear();
-          } else if (chunk.type === 'overlap-speech-started' && agentSpeechStarted) {
-            this.userSpeakingSpan = chunk.userSpeakingSpan;
-            log().debug('overlap speech started, starting interruption inference');
-            overlapSpeechStarted = true;
-            accumulatedSamples = 0;
-            // Include both speech duration and audio prefix duration for context
-            const shiftSize = Math.min(
-              startIdx,
-              Math.round(chunk.speechDuration * this.options.sampleRate) +
-                Math.round(this.options.audioPrefixDuration * this.options.sampleRate),
-            );
-            // Shift the buffer: copy the last `shiftSize` samples before startIdx
-            // to the beginning of the buffer. This preserves recent audio context
-            // (the user's speech that occurred just before overlap was detected).
-            inferenceS16Data.copyWithin(0, startIdx - shiftSize, startIdx);
-            startIdx = shiftSize;
-            cache.clear();
-          } else if (chunk.type === 'overlap-speech-ended') {
-            log().debug('overlap speech ended');
-
-            if (overlapSpeechStarted) {
-              this.userSpeakingSpan = undefined;
-              let latestEntry = Array.from(cache.values()).at(-1);
-              if (!latestEntry) {
-                log().debug('no request made for overlap speech');
-                latestEntry = InterruptionCacheEntry.default();
-              } else {
-                cache.delete(latestEntry.createdAt);
-              }
-              const event: InterruptionEvent = {
-                type: InterruptionEventType.OVERLAP_SPEECH_ENDED,
-                timestamp: Date.now(),
-                isInterruption: false,
-                overlapSpeechStartedAt: this.overlapSpeechStartedAt,
-                speechInput: latestEntry.speechInput,
-                probabilities: latestEntry.probabilities,
-                totalDuration: latestEntry.totalDuration,
-                detectionDelay: latestEntry.detectionDelay,
-                predictionDuration: latestEntry.predictionDuration,
-                probability: latestEntry.probability,
-              };
-              this.eventStream.write(event);
-            }
-          } else if (chunk.type === 'flush') {
-            log().debug('flushing');
-            // do nothing
-          }
-        },
-      },
-      { highWaterMark: Number.MAX_SAFE_INTEGER },
-      { highWaterMark: Number.MAX_SAFE_INTEGER },
-    );
-
-    const httpPostWriter = new WritableStream<Int16Array>(
-      {
-        // Implement the sink
-        write: async (chunk) => {
-          if (!this.overlapSpeechStartedAt) {
-            return;
-          }
-          const resp = await predictHTTP(
-            chunk,
-            { threshold: this.options.threshold, minFrames: this.options.minFrames },
-            {
-              baseUrl: this.options.baseUrl,
-              timeout: this.options.inferenceTimeout,
-              token: await createAccessToken(this.options.apiKey, this.options.apiSecret),
-            },
-          );
-          console.log('received inference response', resp);
-          const { createdAt, isBargein, probabilities, predictionDuration } = resp;
-          const entry = new InterruptionCacheEntry({
-            createdAt,
-            probabilities,
-            isInterruption: isBargein,
-            speechInput: chunk,
-            totalDuration: (performance.now() - createdAt) / 1e9,
-            detectionDelay: Date.now() - this.overlapSpeechStartedAt,
-            predictionDuration,
-          });
-          cache.set(createdAt, entry);
-          if (overlapSpeechStarted && entry.isInterruption) {
-            if (this.userSpeakingSpan) {
-              updateUserSpeakingSpan(this.userSpeakingSpan, entry);
-            }
-            const event: InterruptionEvent = {
-              type: InterruptionEventType.INTERRUPTION,
-              timestamp: Date.now(),
-              overlapSpeechStartedAt: this.overlapSpeechStartedAt,
-              isInterruption: entry.isInterruption,
-              speechInput: entry.speechInput,
-              probabilities: entry.probabilities,
-              totalDuration: entry.totalDuration,
-              predictionDuration: entry.predictionDuration,
-              detectionDelay: entry.detectionDelay,
-              probability: entry.probability,
-            };
-            this.eventStream.write(event);
-          }
-        },
-        close() {
-          console.log('closing http writer');
-        },
-        abort(err) {
-          console.log('Sink error:', err);
-        },
-      },
-      { highWaterMark: Number.MAX_SAFE_INTEGER },
-    );
-
-    this.inputStream.stream().pipeThrough(transformer).pipeTo(httpPostWriter);
-  }
-
-  private ensureInputNotEnded() {
-    if (this.inputStream.closed) {
-      throw new Error('input stream is closed');
-    }
-  }
-
-  private ensureStreamsNotEnded() {
-    this.ensureInputNotEnded();
-  }
-
-  private getResamplerFor(inputSampleRate: number): AudioResampler {
-    if (!this.resampler) {
-      this.resampler = new AudioResampler(inputSampleRate, this.options.sampleRate);
-    }
-    return this.resampler;
-  }
-
-  get stream(): ReadableStream<InterruptionEvent> {
-    return this.eventStream.stream();
-  }
-
-  async pushFrame(frame: InterruptionSentinel | AudioFrame): Promise<void> {
-    this.ensureStreamsNotEnded();
-    if (!(frame instanceof AudioFrame)) {
-      if (frame.type === 'overlap-speech-started') {
-        this.overlapSpeechStartedAt = Date.now() - frame.speechDuration;
-      }
-      return this.inputStream.write(frame);
-    } else if (this.options.sampleRate !== frame.sampleRate) {
-      const resampler = this.getResamplerFor(frame.sampleRate);
-      if (resampler.inputRate !== frame.sampleRate) {
-        throw new Error('the sample rate of the input frames must be consistent');
-      }
-      for (const resampledFrame of resampler.push(frame)) {
-        await this.inputStream.write(resampledFrame);
-      }
-    } else {
-      await this.inputStream.write(frame);
-    }
-  }
-
-  async flush(): Promise<void> {
-    this.ensureStreamsNotEnded();
-    this.inputStream.write(InterruptionStreamSentinel.flush());
-  }
-
-  async endInput(): Promise<void> {
-    await this.flush();
-    await this.inputStream.close();
-  }
-
-  async close(): Promise<void> {
-    if (!this.inputStream.closed) await this.inputStream.close();
-  }
-}
-
-/**
- * Write the audio frame to the output data array and return the new start index
- * and the number of samples written.
- */
-function writeToInferenceS16Data(
-  frame: AudioFrame,
-  startIdx: number,
-  outData: Int16Array,
-  maxAudioDuration: number,
-): { startIdx: number; samplesWritten: number } {
-  const maxWindowSize = Math.floor(maxAudioDuration * frame.sampleRate);
-
-  if (frame.samplesPerChannel > outData.length) {
-    throw new Error('frame samples are greater than the max window size');
-  }
-
-  // Shift the data to the left if the window would overflow
-  const shift = startIdx + frame.samplesPerChannel - maxWindowSize;
-  if (shift > 0) {
-    outData.copyWithin(0, shift, startIdx);
-    startIdx -= shift;
-  }
-
-  // Get the frame data as Int16Array
-  const frameData = new Int16Array(
-    frame.data.buffer,
-    frame.data.byteOffset,
-    frame.samplesPerChannel * frame.channels,
-  );
-
-  if (frame.channels > 1) {
-    // Mix down multiple channels to mono by averaging
-    for (let i = 0; i < frame.samplesPerChannel; i++) {
-      let sum = 0;
-      for (let ch = 0; ch < frame.channels; ch++) {
-        sum += frameData[i * frame.channels + ch] ?? 0;
-      }
-      outData[startIdx + i] = Math.floor(sum / frame.channels);
-    }
-  } else {
-    // Single channel - copy directly
-    outData.set(frameData, startIdx);
-  }
-
-  startIdx += frame.samplesPerChannel;
-  return { startIdx, samplesWritten: frame.samplesPerChannel };
-}

package/src/inference/interruption/defaults.ts

@@ -1,33 +0,0 @@
-import type { InterruptionOptions } from './AdaptiveInterruptionDetector.js';
-import type { ApiConnectOptions } from './InterruptionStream.js';
-
-export const MIN_INTERRUPTION_DURATION = 0.025 * 2; // 25ms per frame, 2 consecutive frames
-export const THRESHOLD = 0.65;
-export const MAX_AUDIO_DURATION = 3.0;
-export const AUDIO_PREFIX_DURATION = 0.5;
-export const DETECTION_INTERVAL = 0.1;
-export const REMOTE_INFERENCE_TIMEOUT = 1.0;
-export const SAMPLE_RATE = 16000;
-export const FRAMES_PER_SECOND = 40;
-export const DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';
-
-export const apiConnectDefaults: ApiConnectOptions = {
-  maxRetries: 3,
-  retryInterval: 2_000,
-  timeout: 10_000,
-} as const;
-
-export const interruptionOptionDefaults: InterruptionOptions = {
-  sampleRate: SAMPLE_RATE,
-  threshold: THRESHOLD,
-  minFrames: Math.ceil(MIN_INTERRUPTION_DURATION * FRAMES_PER_SECOND),
-  maxAudioDuration: MAX_AUDIO_DURATION,
-  audioPrefixDuration: AUDIO_PREFIX_DURATION,
-  detectionInterval: DETECTION_INTERVAL,
-  inferenceTimeout: 10_000,
-  baseUrl: DEFAULT_BASE_URL,
-  apiKey: process.env.LIVEKIT_API_KEY || '',
-  apiSecret: process.env.LIVEKIT_API_SECRET || '',
-  useProxy: false,
-  minInterruptionDuration: MIN_INTERRUPTION_DURATION,
-} as const;

package/src/inference/interruption/http_transport.ts

@@ -1,61 +0,0 @@
-import { ofetch } from 'ofetch';
-
-export interface PostOptions {
-  baseUrl: string;
-  token: string;
-  signal?: AbortSignal;
-  timeout?: number;
-}
-
-export interface PredictOptions {
-  threshold: number;
-  minFrames: number;
-}
-
-export interface PredictEndpointResponse {
-  created_at: number;
-  is_bargein: boolean;
-  probabilities: number[];
-}
-
-export interface PredictResponse {
-  createdAt: number;
-  isBargein: boolean;
-  probabilities: Float32Array;
-  predictionDuration: number;
-}
-
-export async function predictHTTP(
-  data: Int16Array,
-  predictOptions: PredictOptions,
-  options: PostOptions,
-): Promise<PredictResponse> {
-  const createdAt = performance.now();
-  const url = new URL(`/bargein`, options.baseUrl);
-  url.searchParams.append('threshold', predictOptions.threshold.toString());
-  url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());
-  url.searchParams.append('created_at', createdAt.toFixed());
-
-  const { created_at, is_bargein, probabilities } = await ofetch<PredictEndpointResponse>(
-    url.toString(),
-    {
-      retry: 1,
-      retryDelay: 100,
-      headers: {
-        'Content-Type': 'application/octet-stream',
-        Authorization: `Bearer ${options.token}`,
-      },
-      signal: options.signal,
-      timeout: options.timeout,
-      method: 'POST',
-      body: data,
-    },
-  );
-
-  return {
-    createdAt: created_at,
-    isBargein: is_bargein,
-    probabilities: new Float32Array(probabilities),
-    predictionDuration: (performance.now() - createdAt) / 1e9,
-  };
-}

package/src/inference/interruption/index.ts

@@ -1,4 +0,0 @@
-export * from './AdaptiveInterruptionDetector.js';
-export * from './interruption.js';
-export { InterruptionStreamSentinel } from './InterruptionStream.js';
-export type { InterruptionSentinel } from './InterruptionStream.js';

package/src/inference/interruption/interruption.ts

@@ -1,88 +0,0 @@
-import { slidingWindowMinMax } from '../utils.js';
-import { MIN_INTERRUPTION_DURATION } from './defaults.js';
-
-export enum InterruptionEventType {
-  INTERRUPTION = 'interruption',
-  OVERLAP_SPEECH_ENDED = 'overlap_speech_ended',
-}
-export interface InterruptionEvent {
-  type: InterruptionEventType;
-  timestamp: number;
-  isInterruption: boolean;
-  totalDuration: number;
-  predictionDuration: number;
-  detectionDelay: number;
-  overlapSpeechStartedAt?: number;
-  speechInput?: Int16Array;
-  probabilities?: Float32Array;
-  probability: number;
-}
-
-export class InterruptionDetectionError extends Error {
-  readonly type = 'InterruptionDetectionError';
-
-  readonly timestamp: number;
-  readonly label: string;
-  readonly recoverable: boolean;
-
-  constructor(message: string, timestamp: number, label: string, recoverable: boolean) {
-    super(message);
-    this.name = 'InterruptionDetectionError';
-    this.timestamp = timestamp;
-    this.label = label;
-    this.recoverable = recoverable;
-  }
-
-  toString(): string {
-    return `${this.name}: ${this.message} (label=${this.label}, timestamp=${this.timestamp}, recoverable=${this.recoverable})`;
-  }
-}
-
-function estimateProbability(
-  probabilities: Float32Array,
-  windowSize: number = MIN_INTERRUPTION_DURATION,
-): number {
-  const minWindow = Math.ceil(windowSize / 0.025); // 25ms per frame
-  if (probabilities.length < minWindow) {
-    return 0;
-  }
-
-  return slidingWindowMinMax(probabilities, windowSize);
-}
-
-/**
- * Typed cache entry for interruption inference results.
- */
-export class InterruptionCacheEntry {
-  readonly createdAt: number;
-  readonly totalDuration: number;
-  readonly predictionDuration: number;
-  readonly detectionDelay: number;
-  readonly speechInput?: Int16Array;
-  readonly probabilities?: Float32Array;
-  readonly isInterruption?: boolean;
-  readonly probability: number;
-
-  constructor(params: {
-    createdAt: number;
-    speechInput?: Int16Array;
-    totalDuration?: number;
-    predictionDuration?: number;
-    detectionDelay?: number;
-    probabilities?: Float32Array;
-    isInterruption?: boolean;
-  }) {
-    this.createdAt = params.createdAt;
-    this.totalDuration = params.totalDuration ?? 0;
-    this.predictionDuration = params.predictionDuration ?? 0;
-    this.detectionDelay = params.detectionDelay ?? 0;
-    this.speechInput = params.speechInput;
-    this.probabilities = params.probabilities;
-    this.isInterruption = params.isInterruption;
-    this.probability = this.probabilities ? estimateProbability(this.probabilities) : 0;
-  }
-
-  static default(): InterruptionCacheEntry {
-    return new InterruptionCacheEntry({ createdAt: 0 });
-  }
-}

package/src/inference/utils.test.ts

@@ -1,31 +0,0 @@
-// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import { describe, expect, it } from 'vitest';
-import { slidingWindowMinMax } from './utils.js';
-
-describe('slidingWindowMinMax', () => {
-  it('returns -Infinity when array is shorter than window size', () => {
-    expect(slidingWindowMinMax([0.5, 0.6], 3)).toBe(-Infinity);
-    expect(slidingWindowMinMax([], 1)).toBe(-Infinity);
-  });
-
-  it('returns the max value when window size is 1', () => {
-    // With window size 1, min of each window is the element itself,
-    // so max of mins is just the max of the array
-    expect(slidingWindowMinMax([0.1, 0.5, 0.3, 0.8, 0.2], 1)).toBe(0.8);
-  });
-
-  it('finds the best sustained probability across windows', () => {
-    // Windows of size 3: [0.2, 0.8, 0.7], [0.8, 0.7, 0.3], [0.7, 0.3, 0.9]
-    // Mins:              0.2,             0.3,             0.3
-    // Max of mins: 0.3
-    expect(slidingWindowMinMax([0.2, 0.8, 0.7, 0.3, 0.9], 3)).toBe(0.3);
-  });
-
-  it('returns the single element when array length equals window size', () => {
-    // Only one window covering the entire array, return min of that window
-    expect(slidingWindowMinMax([0.5, 0.9, 0.7], 3)).toBe(0.5);
-    expect(slidingWindowMinMax([0.8], 1)).toBe(0.8);
-  });
-});