@livekit/agents 1.0.47 → 1.1.0-dev.0

package/src/inference/interruption/ws_transport.ts

@@ -0,0 +1,402 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { TransformStream } from 'stream/web';
+import WebSocket from 'ws';
+import { z } from 'zod';
+import { log } from '../../log.js';
+import { createAccessToken } from '../utils.js';
+import { intervalForRetry } from './defaults.js';
+import { InterruptionCacheEntry } from './interruption_cache_entry.js';
+import type { OverlappingSpeechEvent } from './types.js';
+import type { BoundedCache } from './utils.js';
+
+// WebSocket message types
+const MSG_SESSION_CREATE = 'session.create';
+const MSG_SESSION_CLOSE = 'session.close';
+const MSG_SESSION_CREATED = 'session.created';
+const MSG_SESSION_CLOSED = 'session.closed';
+const MSG_INTERRUPTION_DETECTED = 'bargein_detected';
+const MSG_INFERENCE_DONE = 'inference_done';
+const MSG_ERROR = 'error';
+
+export interface WsTransportOptions {
+  baseUrl: string;
+  apiKey: string;
+  apiSecret: string;
+  sampleRate: number;
+  threshold: number;
+  minFrames: number;
+  timeout: number;
+  maxRetries?: number;
+}
+
+export interface WsTransportState {
+  overlapSpeechStarted: boolean;
+  overlapSpeechStartedAt: number | undefined;
+  cache: BoundedCache<number, InterruptionCacheEntry>;
+}
+
+const wsMessageSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal(MSG_SESSION_CREATED),
+  }),
+  z.object({
+    type: z.literal(MSG_SESSION_CLOSED),
+  }),
+  z.object({
+    type: z.literal(MSG_INTERRUPTION_DETECTED),
+    created_at: z.number(),
+    probabilities: z.array(z.number()).default([]),
+    prediction_duration: z.number().default(0),
+  }),
+  z.object({
+    type: z.literal(MSG_INFERENCE_DONE),
+    created_at: z.number(),
+    probabilities: z.array(z.number()).default([]),
+    prediction_duration: z.number().default(0),
+    is_bargein: z.boolean().optional(),
+  }),
+  z.object({
+    type: z.literal(MSG_ERROR),
+    message: z.string(),
+    code: z.number().optional(),
+    session_id: z.string().optional(),
+  }),
+]);
+
+type WsMessage = z.infer<typeof wsMessageSchema>;
+
+/**
+ * Creates a WebSocket connection and waits for it to open.
+ */
+async function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {
+  const baseUrl = options.baseUrl.replace(/^http/, 'ws');
+  const token = await createAccessToken(options.apiKey, options.apiSecret);
+  const url = `${baseUrl}/bargein`;
+
+  const ws = new WebSocket(url, {
+    headers: { Authorization: `Bearer ${token}` },
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      ws.terminate();
+      reject(new Error('WebSocket connection timeout'));
+    }, options.timeout);
+    ws.once('open', () => {
+      clearTimeout(timeout);
+      resolve();
+    });
+    ws.once('error', (err: Error) => {
+      clearTimeout(timeout);
+      ws.terminate();
+      reject(err);
+    });
+  });
+
+  return ws;
+}
+
+export interface WsTransportResult {
+  transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;
+  reconnect: () => Promise<void>;
+}
+
+/**
+ * Creates a WebSocket transport TransformStream for interruption detection.
+ *
+ * This transport receives Int16Array audio slices and outputs InterruptionEvents.
+ * It maintains a persistent WebSocket connection with automatic retry on failure.
+ * Returns both the transport and a reconnect function for option updates.
+ */
+export function createWsTransport(
+  options: WsTransportOptions,
+  getState: () => WsTransportState,
+  setState: (partial: Partial<WsTransportState>) => void,
+  updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,
+  onRequestSent?: () => void,
+  getAndResetNumRequests?: () => number,
+): WsTransportResult {
+  const logger = log();
+  let ws: WebSocket | null = null;
+  let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;
+
+  function setupMessageHandler(socket: WebSocket): void {
+    socket.on('message', (data: WebSocket.Data) => {
+      try {
+        const message = wsMessageSchema.parse(JSON.parse(data.toString()));
+        handleMessage(message);
+      } catch {
+        logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');
+      }
+    });
+
+    socket.on('error', (err: Error) => {
+      logger.error({ err }, 'WebSocket error');
+    });
+
+    socket.on('close', (code: number, reason: Buffer) => {
+      logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');
+    });
+  }
+
+  async function ensureConnection(): Promise<void> {
+    if (ws && ws.readyState === WebSocket.OPEN) return;
+
+    const maxRetries = options.maxRetries ?? 3;
+    let lastError: Error | null = null;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        ws = await connectWebSocket(options);
+        setupMessageHandler(ws);
+
+        // Send session.create message
+        const sessionCreateMsg = JSON.stringify({
+          type: MSG_SESSION_CREATE,
+          settings: {
+            sample_rate: options.sampleRate,
+            num_channels: 1,
+            threshold: options.threshold,
+            min_frames: options.minFrames,
+            encoding: 's16le',
+          },
+        });
+        ws.send(sessionCreateMsg);
+        return;
+      } catch (err) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        if (attempt < maxRetries) {
+          const delay = intervalForRetry(attempt);
+          logger.debug(
+            { attempt, delay, err: lastError.message },
+            'WebSocket connection failed, retrying',
+          );
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+    }
+
+    throw lastError ?? new Error('Failed to connect to WebSocket after retries');
+  }
+
+  function handleMessage(message: WsMessage): void {
+    const state = getState();
+
+    switch (message.type) {
+      case MSG_SESSION_CREATED:
+        logger.debug('WebSocket session created');
+        break;
+
+      case MSG_INTERRUPTION_DETECTED: {
+        const createdAt = message.created_at;
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {
+          const existing = state.cache.get(createdAt);
+
+          const totalDurationInS =
+            existing?.requestStartedAt !== undefined
+              ? (performance.now() - existing.requestStartedAt) / 1000
+              : (performance.now() - createdAt) / 1000;
+
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              speechInput: existing?.speechInput,
+              requestStartedAt: existing?.requestStartedAt,
+              totalDurationInS,
+              probabilities: message.probabilities,
+              isInterruption: true,
+              predictionDurationInS: message.prediction_duration,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,
+            },
+          );
+
+          if (updateUserSpeakingSpan) {
+            updateUserSpeakingSpan(entry);
+          }
+
+          logger.debug(
+            {
+              totalDuration: entry.totalDurationInS,
+              predictionDuration: entry.predictionDurationInS,
+              detectionDelay: entry.detectionDelayInS,
+              probability: entry.probability,
+            },
+            'interruption detected',
+          );
+
+          const event: OverlappingSpeechEvent = {
+            type: 'user_overlapping_speech',
+            timestamp: Date.now(),
+            isInterruption: true,
+            totalDurationInS: entry.totalDurationInS,
+            predictionDurationInS: entry.predictionDurationInS,
+            overlapStartedAt: overlapSpeechStartedAt,
+            speechInput: entry.speechInput,
+            probabilities: entry.probabilities,
+            detectionDelayInS: entry.detectionDelayInS,
+            probability: entry.probability,
+            numRequests: getAndResetNumRequests?.() ?? 0,
+          };
+
+          outputController?.enqueue(event);
+          setState({ overlapSpeechStarted: false });
+        }
+        break;
+      }
+
+      case MSG_INFERENCE_DONE: {
+        const createdAt = message.created_at;
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {
+          const existing = state.cache.get(createdAt);
+          const totalDurationInS =
+            existing?.requestStartedAt !== undefined
+              ? (performance.now() - existing.requestStartedAt) / 1000
+              : (performance.now() - createdAt) / 1000;
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              speechInput: existing?.speechInput,
+              requestStartedAt: existing?.requestStartedAt,
+              totalDurationInS,
+              predictionDurationInS: message.prediction_duration,
+              probabilities: message.probabilities,
+              isInterruption: message.is_bargein ?? false,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,
+            },
+          );
+
+          logger.debug(
+            {
+              totalDurationInS: entry.totalDurationInS,
+              predictionDurationInS: entry.predictionDurationInS,
+            },
+            'interruption inference done',
+          );
+        }
+        break;
+      }
+
+      case MSG_SESSION_CLOSED:
+        logger.debug('WebSocket session closed');
+        break;
+
+      case MSG_ERROR:
+        outputController?.error(
+          new Error(
+            `LiveKit Adaptive Interruption error${
+              message.code !== undefined ? ` (${message.code})` : ''
+            }: ${message.message}`,
+          ),
+        );
+        break;
+    }
+  }
+
+  function sendAudioData(audioSlice: Int16Array): void {
+    if (!ws || ws.readyState !== WebSocket.OPEN) {
+      throw new Error('WebSocket not connected');
+    }
+
+    const state = getState();
+    // Use truncated timestamp consistently for both cache key and header
+    // This ensures the server's response created_at matches our cache key
+    const createdAt = Math.floor(performance.now());
+
+    // Store the audio data in cache with truncated timestamp
+    state.cache.set(
+      createdAt,
+      new InterruptionCacheEntry({
+        createdAt,
+        requestStartedAt: performance.now(),
+        speechInput: audioSlice,
+      }),
+    );
+
+    // Create header: 8-byte little-endian uint64 timestamp (milliseconds as integer)
+    const header = new ArrayBuffer(8);
+    const view = new DataView(header);
+    view.setUint32(0, createdAt >>> 0, true);
+    view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);
+
+    // Combine header and audio data
+    const audioBytes = new Uint8Array(
+      audioSlice.buffer,
+      audioSlice.byteOffset,
+      audioSlice.byteLength,
+    );
+    const combined = new Uint8Array(8 + audioBytes.length);
+    combined.set(new Uint8Array(header), 0);
+    combined.set(audioBytes, 8);
+
+    try {
+      ws.send(combined);
+      onRequestSent?.();
+    } catch (e: unknown) {
+      logger.error(e, `failed to send audio via websocket`);
+    }
+  }
+
+  function close(): void {
+    if (ws?.readyState === WebSocket.OPEN) {
+      const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });
+      try {
+        ws.send(closeMsg);
+      } catch (e: unknown) {
+        logger.error(e, 'failed to send close message');
+      }
+    }
+    ws?.close(1000); // signal normal websocket closure
+    ws = null;
+  }
+
+  /**
+   * Reconnect the WebSocket with updated options.
+   * This is called when options are updated via updateOptions().
+   */
+  async function reconnect(): Promise<void> {
+    close();
+  }
+
+  const transport = new TransformStream<
+    Int16Array | OverlappingSpeechEvent,
+    OverlappingSpeechEvent
+  >(
+    {
+      async start(controller) {
+        outputController = controller;
+        await ensureConnection();
+      },
+
+      transform(chunk, controller) {
+        if (!(chunk instanceof Int16Array)) {
+          controller.enqueue(chunk);
+          return;
+        }
+
+        // Only forwards buffered audio while overlap speech is actively on.
+        const state = getState();
+        if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;
+
+        try {
+          sendAudioData(chunk);
+        } catch (err) {
+          logger.error({ err }, 'Failed to send audio data over WebSocket');
+        }
+      },
+
+      flush() {
+        close();
+      },
+    },
+    { highWaterMark: 2 },
+    { highWaterMark: 2 },
+  );
+
+  return { transport, reconnect };
+}

package/src/inference/llm.ts

@@ -2,19 +2,12 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import OpenAI from 'openai';
-import {
-  APIConnectionError,
-  APIStatusError,
-  APITimeoutError,
-  DEFAULT_API_CONNECT_OPTIONS,
-  type Expand,
-  toError,
-} from '../index.js';
+import { APIConnectionError, APIStatusError, APITimeoutError } from '../_exceptions.js';
 import * as llm from '../llm/index.js';
+import { DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
 import type { APIConnectOptions } from '../types.js';
-import { type AnyString, createAccessToken } from './utils.js';
-
-const DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';
+import { type Expand, toError } from '../utils.js';
+import { type AnyString, createAccessToken, getDefaultInferenceUrl } from './utils.js';
 
 export type OpenAIModels =
   | 'openai/gpt-5.2'
@@ -127,7 +120,7 @@ export class LLM extends llm.LLM {
       strictToolSchema = false,
     } = opts;
 
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || getDefaultInferenceUrl();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');
@@ -163,6 +156,10 @@ export class LLM extends llm.LLM {
     return this.opts.model;
   }
 
+  get provider(): string {
+    return 'livekit';
+  }
+
   static fromModelString(modelString: string): LLM {
     return new LLM({ model: modelString });
   }

package/src/inference/stt.ts

@@ -22,7 +22,7 @@ import {
   type SttTranscriptEvent,
   sttServerEventSchema,
 } from './api_protos.js';
-import { type AnyString, connectWs, createAccessToken } from './utils.js';
+import { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js';
 
 export type DeepgramModels =
   | 'deepgram/flux-general'
@@ -151,7 +151,6 @@ export type STTEncoding = 'pcm_s16le';
 
 const DEFAULT_ENCODING: STTEncoding = 'pcm_s16le';
 const DEFAULT_SAMPLE_RATE = 16000;
-const DEFAULT_BASE_URL = 'wss://agent-gateway.livekit.cloud/v1';
 const DEFAULT_CANCEL_TIMEOUT = 5000;
 
 export interface InferenceSTTOptions<TModel extends STTModels> {
@@ -203,7 +202,7 @@ export class STT<TModel extends STTModels> extends BaseSTT {
       connOptions,
     } = opts || {};
 
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || getDefaultInferenceUrl();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');
@@ -253,6 +252,14 @@ export class STT<TModel extends STTModels> extends BaseSTT {
     return 'inference.STT';
   }
 
+  get model(): string {
+    return this.opts.model ?? 'auto';
+  }
+
+  get provider(): string {
+    return 'livekit';
+  }
+
   static fromModelString(modelString: string): STT<AnyString> {
     const [model, language] = parseSTTModelString(modelString);
     return new STT({ model, language });

package/src/inference/tts.ts

@@ -19,7 +19,7 @@ import {
   ttsClientEventSchema,
   ttsServerEventSchema,
 } from './api_protos.js';
-import { type AnyString, connectWs, createAccessToken } from './utils.js';
+import { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js';
 
 export type CartesiaModels =
   | 'cartesia/sonic-3'
@@ -136,7 +136,6 @@ type TTSEncoding = 'pcm_s16le';
 
 const DEFAULT_ENCODING: TTSEncoding = 'pcm_s16le';
 const DEFAULT_SAMPLE_RATE = 16000;
-const DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';
 const NUM_CHANNELS = 1;
 const DEFAULT_LANGUAGE = 'en';
 
@@ -193,7 +192,7 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
       connOptions,
     } = opts || {};
 
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || getDefaultInferenceUrl();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');
@@ -254,6 +253,14 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
     return 'inference.TTS';
   }
 
+  get model(): string {
+    return this.opts.model ?? 'unknown';
+  }
+
+  get provider(): string {
+    return 'livekit';
+  }
+
   static fromModelString(modelString: string): TTS<AnyString> {
     const [model, voice] = parseTTSModelString(modelString);
     return new TTS({ model, voice: voice || undefined });

package/src/inference/utils.ts

@@ -3,10 +3,38 @@
 // SPDX-License-Identifier: Apache-2.0
 import { AccessToken } from 'livekit-server-sdk';
 import { WebSocket } from 'ws';
-import { APIConnectionError, APIStatusError } from '../index.js';
+import { APIConnectionError, APIStatusError } from '../_exceptions.js';
 
 export type AnyString = string & NonNullable<unknown>;
 
+/** Default production inference URL */
+export const DEFAULT_INFERENCE_URL = 'https://agent-gateway.livekit.cloud/v1';
+
+/** Staging inference URL */
+export const STAGING_INFERENCE_URL = 'https://agent-gateway.staging.livekit.cloud/v1';
+
+/**
+ * Get the default inference URL based on the environment.
+ *
+ * Priority:
+ * 1. LIVEKIT_INFERENCE_URL if set
+ * 2. If LIVEKIT_URL contains '.staging.livekit.cloud', use staging gateway
+ * 3. Otherwise, use production gateway
+ */
+export function getDefaultInferenceUrl(): string {
+  const inferenceUrl = process.env.LIVEKIT_INFERENCE_URL;
+  if (inferenceUrl) {
+    return inferenceUrl;
+  }
+
+  const livekitUrl = process.env.LIVEKIT_URL || '';
+  if (livekitUrl.includes('.staging.livekit.cloud')) {
+    return STAGING_INFERENCE_URL;
+  }
+
+  return DEFAULT_INFERENCE_URL;
+}
+
 export async function createAccessToken(
   apiKey: string,
   apiSecret: string,

package/src/llm/chat_context.test.ts

@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { describe, expect, it } from 'vitest';
+import { initializeLogger } from '../log.js';
+import { FakeLLM } from '../voice/testing/fake_llm.js';
 import {
   type AudioContent,
   ChatContext,
@@ -13,6 +15,8 @@ import {
   ReadonlyChatContext,
 } from './chat_context.js';
 
+initializeLogger({ pretty: false, level: 'error' });
+
 describe('ChatContext.toJSON', () => {
   it('should match snapshot for empty context', () => {
     const context = new ChatContext();
@@ -283,6 +287,50 @@ describe('ChatContext.toJSON', () => {
   });
 });
 
+describe('ChatContext._summarize', () => {
+  it('keeps chronological timestamps with summary + tail', async () => {
+    const ctx = new ChatContext();
+    ctx.addMessage({ role: 'system', content: 'System prompt', createdAt: 0 });
+    ctx.addMessage({ role: 'user', content: 'hello', createdAt: 1000 });
+    ctx.addMessage({ role: 'assistant', content: 'hi there', createdAt: 2000 });
+    ctx.insert(
+      new FunctionCallOutput({
+        callId: 'call_1',
+        name: 'lookup',
+        output: '{"ok":true}',
+        isError: false,
+        createdAt: 3500,
+      }),
+    );
+    ctx.addMessage({ role: 'user', content: 'my color is blue', createdAt: 3000 });
+    ctx.addMessage({ role: 'assistant', content: 'noted', createdAt: 4000 });
+
+    const fake = new FakeLLM([
+      {
+        input: 'Conversation to summarize:\n\nuser: hello\nassistant: hi there',
+        content: 'condensed head',
+      },
+    ]);
+
+    await ctx._summarize(fake, { keepLastTurns: 1 });
+
+    const summary = ctx.items.find(
+      (item) =>
+        item.type === 'message' && item.role === 'assistant' && item.extra?.is_summary === true,
+    );
+    expect(summary).toBeDefined();
+    if (!summary || summary.type !== 'message') {
+      throw new Error('summary message is missing');
+    }
+
+    expect(summary.createdAt).toBeCloseTo(2999.999, 6);
+
+    const createdAts = ctx.items.map((item) => item.createdAt);
+    const sorted = [...createdAts].sort((a, b) => a - b);
+    expect(createdAts).toEqual(sorted);
+  });
+});
+
 describe('ReadonlyChatContext with immutable array', () => {
   it('should have readonly property set to true', () => {
     const items: ChatItem[] = [