@livekit/agents 1.0.34 → 1.0.36-dev.0

package/src/inference/interruption/AdaptiveInterruptionDetector.ts

@@ -0,0 +1,166 @@
+import type { TypedEventEmitter } from '@livekit/typed-emitter';
+import EventEmitter from 'events';
+import { type ReadableStream, TransformStream } from 'stream/web';
+import { InterruptionStreamBase } from './InterruptionStream.js';
+import {
+  DEFAULT_BASE_URL,
+  FRAMES_PER_SECOND,
+  SAMPLE_RATE,
+  interruptionOptionDefaults,
+} from './defaults.js';
+import {
+  type InterruptionDetectionError,
+  type InterruptionEvent,
+  InterruptionEventType,
+} from './interruption.js';
+
+type InterruptionCallbacks = {
+  interruptionDetected: () => void;
+  overlapSpeechDetected: () => void;
+  error: (error: InterruptionDetectionError) => void;
+};
+
+export interface InterruptionOptions {
+  sampleRate: number;
+  threshold: number;
+  minFrames: number;
+  maxAudioDuration: number;
+  audioPrefixDuration: number;
+  detectionInterval: number;
+  inferenceTimeout: number;
+  minInterruptionDuration: number;
+  baseUrl: string;
+  apiKey: string;
+  apiSecret: string;
+  useProxy: boolean;
+}
+
+export type AdaptiveInterruptionDetectorOptions = Partial<InterruptionOptions>;
+
+export class AdaptiveInterruptionDetector extends (EventEmitter as new () => TypedEventEmitter<InterruptionCallbacks>) {
+  options: InterruptionOptions;
+  private label: string;
+  private streams: WeakSet<object>; // TODO: Union of InterruptionHttpStream | InterruptionWebSocketStream
+
+  constructor(options: AdaptiveInterruptionDetectorOptions = {}) {
+    super();
+
+    const {
+      maxAudioDuration,
+      baseUrl,
+      apiKey,
+      apiSecret,
+      useProxy: useProxyArg,
+      audioPrefixDuration,
+      threshold,
+      detectionInterval,
+      inferenceTimeout,
+      minInterruptionDuration,
+    } = { ...interruptionOptionDefaults, ...options };
+
+    if (maxAudioDuration > 3.0) {
+      throw new Error('maxAudioDuration must be less than or equal to 3.0 seconds');
+    }
+
+    const lkBaseUrl = baseUrl ?? process.env.LIVEKIT_REMOTE_EOT_URL ?? DEFAULT_BASE_URL;
+    let lkApiKey = apiKey ?? '';
+    let lkApiSecret = apiSecret ?? '';
+    let useProxy: boolean;
+
+    // use LiveKit credentials if using the default base URL (inference)
+    if (lkBaseUrl === DEFAULT_BASE_URL) {
+      lkApiKey =
+        apiKey ?? process.env.LIVEKIT_INFERENCE_API_KEY ?? process.env.LIVEKIT_API_KEY ?? '';
+      if (!lkApiKey) {
+        throw new Error(
+          'apiKey is required, either as argument or set LIVEKIT_API_KEY environmental variable',
+        );
+      }
+
+      lkApiSecret =
+        apiSecret ??
+        process.env.LIVEKIT_INFERENCE_API_SECRET ??
+        process.env.LIVEKIT_API_SECRET ??
+        '';
+      if (!lkApiSecret) {
+        throw new Error(
+          'apiSecret is required, either as argument or set LIVEKIT_API_SECRET environmental variable',
+        );
+      }
+
+      useProxy = true;
+    } else {
+      useProxy = useProxyArg ?? false;
+    }
+
+    this.options = {
+      sampleRate: SAMPLE_RATE,
+      threshold,
+      minFrames: Math.ceil(minInterruptionDuration * FRAMES_PER_SECOND),
+      maxAudioDuration,
+      audioPrefixDuration,
+      detectionInterval,
+      inferenceTimeout,
+      baseUrl: lkBaseUrl,
+      apiKey: lkApiKey,
+      apiSecret: lkApiSecret,
+      useProxy,
+      minInterruptionDuration,
+    };
+
+    this.label = `${this.constructor.name}`;
+    this.streams = new WeakSet();
+
+    console.info('adaptive interruption detector initialized', {
+      baseUrl: this.options.baseUrl,
+      detectionInterval: this.options.detectionInterval,
+      audioPrefixDuration: this.options.audioPrefixDuration,
+      maxAudioDuration: this.options.maxAudioDuration,
+      minFrames: this.options.minFrames,
+      threshold: this.options.threshold,
+      inferenceTimeout: this.options.inferenceTimeout,
+      useProxy: this.options.useProxy,
+    });
+  }
+
+  /**
+   * Creates a new InterruptionStreamBase for internal use.
+   * The stream can receive audio frames and sentinels via pushFrame().
+   * Use this when you need direct access to the stream for pushing frames.
+   */
+  createStream(): InterruptionStreamBase {
+    const stream = new InterruptionStreamBase(this, {});
+    this.streams.add(stream);
+    return stream;
+  }
+
+  /**
+   * Creates a new interruption stream and returns a ReadableStream of InterruptionEvents.
+   * This is a convenience method for consuming interruption events without needing
+   * to manage the underlying stream directly.
+   */
+  stream(): ReadableStream<InterruptionEvent> {
+    const httpStream = this.createStream();
+    const transformer = new TransformStream<InterruptionEvent, InterruptionEvent>({
+      transform: (chunk, controller) => {
+        if (chunk.type === InterruptionEventType.INTERRUPTION) {
+          this.emit('interruptionDetected'); // TODO payload
+        } else if (chunk.type === InterruptionEventType.OVERLAP_SPEECH_ENDED) {
+          this.emit('overlapSpeechDetected'); // TODO payload
+        }
+        controller.enqueue(chunk);
+      },
+    });
+    const stream = httpStream.stream.pipeThrough(transformer);
+    return stream;
+  }
+
+  updateOptions(options: { threshold?: number; minInterruptionDuration?: number }): void {
+    if (options.threshold !== undefined) {
+      this.options.threshold = options.threshold;
+    }
+    if (options.minInterruptionDuration !== undefined) {
+      this.options.minFrames = Math.ceil(options.minInterruptionDuration * FRAMES_PER_SECOND);
+    }
+  }
+}

package/src/inference/interruption/InterruptionStream.ts

@@ -0,0 +1,397 @@
+import { AudioFrame, AudioResampler } from '@livekit/rtc-node';
+import type { Span } from '@opentelemetry/api';
+import { traceTypes } from '../../telemetry/index.js';
+import { type ReadableStream, TransformStream, WritableStream } from 'stream/web';
+import { log } from '../../log.js';
+import { type StreamChannel, createStreamChannel } from '../../stream/stream_channel.js';
+import { createAccessToken } from '../utils.js';
+import type {
+  AdaptiveInterruptionDetector,
+  InterruptionOptions,
+} from './AdaptiveInterruptionDetector.js';
+import { apiConnectDefaults } from './defaults.js';
+import { predictHTTP } from './http_transport.js';
+import {
+  InterruptionCacheEntry,
+  type InterruptionDetectionError,
+  type InterruptionEvent,
+  InterruptionEventType,
+} from './interruption.js';
+
+export interface AgentSpeechStarted {
+  type: 'agent-speech-started';
+}
+
+export interface AgentSpeechEnded {
+  type: 'agent-speech-ended';
+}
+
+export interface OverlapSpeechStarted {
+  type: 'overlap-speech-started';
+  speechDuration: number;
+  userSpeakingSpan: Span;
+}
+
+export interface OverlapSpeechEnded {
+  type: 'overlap-speech-ended';
+}
+
+export interface Flush {
+  type: 'flush';
+}
+
+export type InterruptionSentinel =
+  | AgentSpeechStarted
+  | AgentSpeechEnded
+  | OverlapSpeechStarted
+  | OverlapSpeechEnded
+  | Flush;
+
+export class InterruptionStreamSentinel {
+  static speechStarted(): AgentSpeechStarted {
+    return { type: 'agent-speech-started' };
+  }
+
+  static speechEnded(): AgentSpeechEnded {
+    return { type: 'agent-speech-ended' };
+  }
+
+  static overlapSpeechStarted(
+    speechDuration: number,
+    userSpeakingSpan: Span,
+  ): OverlapSpeechStarted {
+    return { type: 'overlap-speech-started', speechDuration, userSpeakingSpan };
+  }
+
+  static overlapSpeechEnded(): OverlapSpeechEnded {
+    return { type: 'overlap-speech-ended' };
+  }
+
+  static flush(): Flush {
+    return { type: 'flush' };
+  }
+}
+
+export interface ApiConnectOptions {
+  maxRetries: number;
+  retryInterval: number;
+  timeout: number;
+}
+
+function updateUserSpeakingSpan(span: Span, entry: InterruptionCacheEntry) {
+  span.setAttribute(
+    traceTypes.ATTR_IS_INTERRUPTION,
+    (entry.isInterruption ?? false).toString().toLowerCase(),
+  );
+  span.setAttribute(traceTypes.ATTR_INTERRUPTION_PROBABILITY, entry.probability);
+  span.setAttribute(traceTypes.ATTR_INTERRUPTION_TOTAL_DURATION, entry.totalDuration);
+  span.setAttribute(traceTypes.ATTR_INTERRUPTION_PREDICTION_DURATION, entry.predictionDuration);
+  span.setAttribute(traceTypes.ATTR_INTERRUPTION_DETECTION_DELAY, entry.detectionDelay);
+}
+
+export class InterruptionStreamBase {
+  private inputStream: StreamChannel<InterruptionSentinel | AudioFrame, InterruptionDetectionError>;
+
+  private eventStream: StreamChannel<InterruptionEvent, InterruptionDetectionError>;
+
+  private resampler?: AudioResampler;
+
+  private userSpeakingSpan: Span | undefined;
+
+  private overlapSpeechStartedAt: number | undefined;
+
+  private options: InterruptionOptions;
+
+  private apiOptions: ApiConnectOptions;
+
+  private model: AdaptiveInterruptionDetector;
+
+  constructor(model: AdaptiveInterruptionDetector, apiOptions: Partial<ApiConnectOptions>) {
+    this.inputStream = createStreamChannel<
+      InterruptionSentinel | AudioFrame,
+      InterruptionDetectionError
+    >();
+
+    this.eventStream = createStreamChannel<InterruptionEvent, InterruptionDetectionError>();
+
+    this.model = model;
+    this.options = model.options;
+    this.apiOptions = { ...apiConnectDefaults, ...apiOptions };
+
+    this.setupTransform();
+  }
+
+  private setupTransform() {
+    let agentSpeechStarted = false;
+    let startIdx = 0;
+    let accumulatedSamples = 0;
+    let overlapSpeechStarted = false;
+    const cache = new Map<number, InterruptionCacheEntry>(); // TODO limit cache size
+    const inferenceS16Data = new Int16Array(
+      Math.ceil(this.options.maxAudioDuration * this.options.sampleRate),
+    ).fill(0);
+
+    const transformer = new TransformStream<InterruptionSentinel | AudioFrame, Int16Array>(
+      {
+        transform: (chunk, controller) => {
+          if (chunk instanceof AudioFrame) {
+            if (!agentSpeechStarted) {
+              return;
+            }
+            if (this.options.sampleRate !== chunk.sampleRate) {
+              controller.error('the sample rate of the input frames must be consistent');
+              return;
+            }
+            const result = writeToInferenceS16Data(
+              chunk,
+              startIdx,
+              inferenceS16Data,
+              this.options.maxAudioDuration,
+            );
+            startIdx = result.startIdx;
+            accumulatedSamples += result.samplesWritten;
+
+            // Send data for inference when enough samples accumulated during overlap
+            if (
+              accumulatedSamples >=
+                Math.floor(this.options.detectionInterval * this.options.sampleRate) &&
+              overlapSpeechStarted
+            ) {
+              // Send a copy of the audio data up to startIdx for inference
+              const audioSlice = inferenceS16Data.slice(0, startIdx);
+              // TODO: send to data channel - dataChan.send(audioSlice);
+              accumulatedSamples = 0;
+              controller.enqueue(audioSlice);
+            }
+          } else if (chunk.type === 'agent-speech-started') {
+            log().debug('agent speech started');
+
+            agentSpeechStarted = true;
+            overlapSpeechStarted = false;
+            accumulatedSamples = 0;
+            startIdx = 0;
+            cache.clear();
+          } else if (chunk.type === 'agent-speech-ended') {
+            log().debug('agent speech ended');
+
+            agentSpeechStarted = false;
+            overlapSpeechStarted = false;
+            accumulatedSamples = 0;
+            startIdx = 0;
+            cache.clear();
+          } else if (chunk.type === 'overlap-speech-started' && agentSpeechStarted) {
+            this.userSpeakingSpan = chunk.userSpeakingSpan;
+            log().debug('overlap speech started, starting interruption inference');
+            overlapSpeechStarted = true;
+            accumulatedSamples = 0;
+            // Include both speech duration and audio prefix duration for context
+            const shiftSize = Math.min(
+              startIdx,
+              Math.round(chunk.speechDuration * this.options.sampleRate) +
+                Math.round(this.options.audioPrefixDuration * this.options.sampleRate),
+            );
+            // Shift the buffer: copy the last `shiftSize` samples before startIdx
+            // to the beginning of the buffer. This preserves recent audio context
+            // (the user's speech that occurred just before overlap was detected).
+            inferenceS16Data.copyWithin(0, startIdx - shiftSize, startIdx);
+            startIdx = shiftSize;
+            cache.clear();
+          } else if (chunk.type === 'overlap-speech-ended') {
+            log().debug('overlap speech ended');
+
+            if (overlapSpeechStarted) {
+              this.userSpeakingSpan = undefined;
+              let latestEntry = Array.from(cache.values()).at(-1);
+              if (!latestEntry) {
+                log().debug('no request made for overlap speech');
+                latestEntry = InterruptionCacheEntry.default();
+              } else {
+                cache.delete(latestEntry.createdAt);
+              }
+              const event: InterruptionEvent = {
+                type: InterruptionEventType.OVERLAP_SPEECH_ENDED,
+                timestamp: Date.now(),
+                isInterruption: false,
+                overlapSpeechStartedAt: this.overlapSpeechStartedAt,
+                speechInput: latestEntry.speechInput,
+                probabilities: latestEntry.probabilities,
+                totalDuration: latestEntry.totalDuration,
+                detectionDelay: latestEntry.detectionDelay,
+                predictionDuration: latestEntry.predictionDuration,
+                probability: latestEntry.probability,
+              };
+              this.eventStream.write(event);
+            }
+          } else if (chunk.type === 'flush') {
+            log().debug('flushing');
+            // do nothing
+          }
+        },
+      },
+      { highWaterMark: Number.MAX_SAFE_INTEGER },
+      { highWaterMark: Number.MAX_SAFE_INTEGER },
+    );
+
+    const httpPostWriter = new WritableStream<Int16Array>(
+      {
+        // Implement the sink
+        write: async (chunk) => {
+          if (!this.overlapSpeechStartedAt) {
+            return;
+          }
+          const resp = await predictHTTP(
+            chunk,
+            { threshold: this.options.threshold, minFrames: this.options.minFrames },
+            {
+              baseUrl: this.options.baseUrl,
+              timeout: this.options.inferenceTimeout,
+              token: await createAccessToken(this.options.apiKey, this.options.apiSecret),
+            },
+          );
+          console.log('received inference response', resp);
+          const { createdAt, isBargein, probabilities, predictionDuration } = resp;
+          const entry = new InterruptionCacheEntry({
+            createdAt,
+            probabilities,
+            isInterruption: isBargein,
+            speechInput: chunk,
+            totalDuration: (performance.now() - createdAt) / 1e9,
+            detectionDelay: Date.now() - this.overlapSpeechStartedAt,
+            predictionDuration,
+          });
+          cache.set(createdAt, entry);
+          if (overlapSpeechStarted && entry.isInterruption) {
+            if (this.userSpeakingSpan) {
+              updateUserSpeakingSpan(this.userSpeakingSpan, entry);
+            }
+            const event: InterruptionEvent = {
+              type: InterruptionEventType.INTERRUPTION,
+              timestamp: Date.now(),
+              overlapSpeechStartedAt: this.overlapSpeechStartedAt,
+              isInterruption: entry.isInterruption,
+              speechInput: entry.speechInput,
+              probabilities: entry.probabilities,
+              totalDuration: entry.totalDuration,
+              predictionDuration: entry.predictionDuration,
+              detectionDelay: entry.detectionDelay,
+              probability: entry.probability,
+            };
+            this.eventStream.write(event);
+          }
+        },
+        close() {
+          console.log('closing http writer');
+        },
+        abort(err) {
+          console.log('Sink error:', err);
+        },
+      },
+      { highWaterMark: Number.MAX_SAFE_INTEGER },
+    );
+
+    this.inputStream.stream().pipeThrough(transformer).pipeTo(httpPostWriter);
+  }
+
+  private ensureInputNotEnded() {
+    if (this.inputStream.closed) {
+      throw new Error('input stream is closed');
+    }
+  }
+
+  private ensureStreamsNotEnded() {
+    this.ensureInputNotEnded();
+  }
+
+  private getResamplerFor(inputSampleRate: number): AudioResampler {
+    if (!this.resampler) {
+      this.resampler = new AudioResampler(inputSampleRate, this.options.sampleRate);
+    }
+    return this.resampler;
+  }
+
+  get stream(): ReadableStream<InterruptionEvent> {
+    return this.eventStream.stream();
+  }
+
+  async pushFrame(frame: InterruptionSentinel | AudioFrame): Promise<void> {
+    this.ensureStreamsNotEnded();
+    if (!(frame instanceof AudioFrame)) {
+      if (frame.type === 'overlap-speech-started') {
+        this.overlapSpeechStartedAt = Date.now() - frame.speechDuration;
+      }
+      return this.inputStream.write(frame);
+    } else if (this.options.sampleRate !== frame.sampleRate) {
+      const resampler = this.getResamplerFor(frame.sampleRate);
+      if (resampler.inputRate !== frame.sampleRate) {
+        throw new Error('the sample rate of the input frames must be consistent');
+      }
+      for (const resampledFrame of resampler.push(frame)) {
+        await this.inputStream.write(resampledFrame);
+      }
+    } else {
+      await this.inputStream.write(frame);
+    }
+  }
+
+  async flush(): Promise<void> {
+    this.ensureStreamsNotEnded();
+    this.inputStream.write(InterruptionStreamSentinel.flush());
+  }
+
+  async endInput(): Promise<void> {
+    await this.flush();
+    await this.inputStream.close();
+  }
+
+  async close(): Promise<void> {
+    if (!this.inputStream.closed) await this.inputStream.close();
+  }
+}
+
+/**
+ * Write the audio frame to the output data array and return the new start index
+ * and the number of samples written.
+ */
+function writeToInferenceS16Data(
+  frame: AudioFrame,
+  startIdx: number,
+  outData: Int16Array,
+  maxAudioDuration: number,
+): { startIdx: number; samplesWritten: number } {
+  const maxWindowSize = Math.floor(maxAudioDuration * frame.sampleRate);
+
+  if (frame.samplesPerChannel > outData.length) {
+    throw new Error('frame samples are greater than the max window size');
+  }
+
+  // Shift the data to the left if the window would overflow
+  const shift = startIdx + frame.samplesPerChannel - maxWindowSize;
+  if (shift > 0) {
+    outData.copyWithin(0, shift, startIdx);
+    startIdx -= shift;
+  }
+
+  // Get the frame data as Int16Array
+  const frameData = new Int16Array(
+    frame.data.buffer,
+    frame.data.byteOffset,
+    frame.samplesPerChannel * frame.channels,
+  );
+
+  if (frame.channels > 1) {
+    // Mix down multiple channels to mono by averaging
+    for (let i = 0; i < frame.samplesPerChannel; i++) {
+      let sum = 0;
+      for (let ch = 0; ch < frame.channels; ch++) {
+        sum += frameData[i * frame.channels + ch] ?? 0;
+      }
+      outData[startIdx + i] = Math.floor(sum / frame.channels);
+    }
+  } else {
+    // Single channel - copy directly
+    outData.set(frameData, startIdx);
+  }
+
+  startIdx += frame.samplesPerChannel;
+  return { startIdx, samplesWritten: frame.samplesPerChannel };
+}

package/src/inference/interruption/defaults.ts

@@ -0,0 +1,33 @@
+import type { InterruptionOptions } from './AdaptiveInterruptionDetector.js';
+import type { ApiConnectOptions } from './InterruptionStream.js';
+
+export const MIN_INTERRUPTION_DURATION = 0.025 * 2; // 25ms per frame, 2 consecutive frames
+export const THRESHOLD = 0.65;
+export const MAX_AUDIO_DURATION = 3.0;
+export const AUDIO_PREFIX_DURATION = 0.5;
+export const DETECTION_INTERVAL = 0.1;
+export const REMOTE_INFERENCE_TIMEOUT = 1.0;
+export const SAMPLE_RATE = 16000;
+export const FRAMES_PER_SECOND = 40;
+export const DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';
+
+export const apiConnectDefaults: ApiConnectOptions = {
+  maxRetries: 3,
+  retryInterval: 2_000,
+  timeout: 10_000,
+} as const;
+
+export const interruptionOptionDefaults: InterruptionOptions = {
+  sampleRate: SAMPLE_RATE,
+  threshold: THRESHOLD,
+  minFrames: Math.ceil(MIN_INTERRUPTION_DURATION * FRAMES_PER_SECOND),
+  maxAudioDuration: MAX_AUDIO_DURATION,
+  audioPrefixDuration: AUDIO_PREFIX_DURATION,
+  detectionInterval: DETECTION_INTERVAL,
+  inferenceTimeout: 10_000,
+  baseUrl: DEFAULT_BASE_URL,
+  apiKey: process.env.LIVEKIT_API_KEY || '',
+  apiSecret: process.env.LIVEKIT_API_SECRET || '',
+  useProxy: false,
+  minInterruptionDuration: MIN_INTERRUPTION_DURATION,
+} as const;

package/src/inference/interruption/http_transport.ts

@@ -0,0 +1,61 @@
+import { ofetch } from 'ofetch';
+
+export interface PostOptions {
+  baseUrl: string;
+  token: string;
+  signal?: AbortSignal;
+  timeout?: number;
+}
+
+export interface PredictOptions {
+  threshold: number;
+  minFrames: number;
+}
+
+export interface PredictEndpointResponse {
+  created_at: number;
+  is_bargein: boolean;
+  probabilities: number[];
+}
+
+export interface PredictResponse {
+  createdAt: number;
+  isBargein: boolean;
+  probabilities: Float32Array;
+  predictionDuration: number;
+}
+
+export async function predictHTTP(
+  data: Int16Array,
+  predictOptions: PredictOptions,
+  options: PostOptions,
+): Promise<PredictResponse> {
+  const createdAt = performance.now();
+  const url = new URL(`/bargein`, options.baseUrl);
+  url.searchParams.append('threshold', predictOptions.threshold.toString());
+  url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());
+  url.searchParams.append('created_at', createdAt.toFixed());
+
+  const { created_at, is_bargein, probabilities } = await ofetch<PredictEndpointResponse>(
+    url.toString(),
+    {
+      retry: 1,
+      retryDelay: 100,
+      headers: {
+        'Content-Type': 'application/octet-stream',
+        Authorization: `Bearer ${options.token}`,
+      },
+      signal: options.signal,
+      timeout: options.timeout,
+      method: 'POST',
+      body: data,
+    },
+  );
+
+  return {
+    createdAt: created_at,
+    isBargein: is_bargein,
+    probabilities: new Float32Array(probabilities),
+    predictionDuration: (performance.now() - createdAt) / 1e9,
+  };
+}

package/src/inference/interruption/index.ts

@@ -0,0 +1,4 @@
+export * from './AdaptiveInterruptionDetector.js';
+export * from './interruption.js';
+export { InterruptionStreamSentinel } from './InterruptionStream.js';
+export type { InterruptionSentinel } from './InterruptionStream.js';