@livekit/agents 1.0.42 → 1.0.44

package/src/inference/tts.test.ts

@@ -0,0 +1,255 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { beforeAll, describe, expect, it } from 'vitest';
+import { initializeLogger } from '../log.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { TTS, type TTSFallbackModel, normalizeTTSFallback, parseTTSModelString } from './tts.js';
+
+beforeAll(() => {
+  initializeLogger({ level: 'silent', pretty: false });
+});
+
+/** Helper to create TTS with required credentials. */
+function makeTts(overrides: Record<string, unknown> = {}) {
+  const defaults = {
+    model: 'cartesia/sonic' as const,
+    apiKey: 'test-key',
+    apiSecret: 'test-secret',
+    baseURL: 'https://example.livekit.cloud',
+  };
+  return new TTS({ ...defaults, ...overrides });
+}
+
+describe('parseTTSModelString', () => {
+  it('simple model without voice', () => {
+    const [model, voice] = parseTTSModelString('cartesia');
+    expect(model).toBe('cartesia');
+    expect(voice).toBeUndefined();
+  });
+
+  it('model with voice suffix', () => {
+    const [model, voice] = parseTTSModelString('cartesia:my-voice-id');
+    expect(model).toBe('cartesia');
+    expect(voice).toBe('my-voice-id');
+  });
+
+  it('provider/model format without voice', () => {
+    const [model, voice] = parseTTSModelString('cartesia/sonic');
+    expect(model).toBe('cartesia/sonic');
+    expect(voice).toBeUndefined();
+  });
+
+  it('provider/model format with voice', () => {
+    const [model, voice] = parseTTSModelString('cartesia/sonic:my-voice-id');
+    expect(model).toBe('cartesia/sonic');
+    expect(voice).toBe('my-voice-id');
+  });
+
+  it.each([
+    ['elevenlabs/eleven_flash_v2:voice123', 'elevenlabs/eleven_flash_v2', 'voice123'],
+    ['rime:speaker-a', 'rime', 'speaker-a'],
+    ['rime/mist:narrator', 'rime/mist', 'narrator'],
+    ['inworld/inworld-tts-1:character', 'inworld/inworld-tts-1', 'character'],
+    ['cartesia/sonic-turbo:deep-voice', 'cartesia/sonic-turbo', 'deep-voice'],
+  ])('various providers and voices: %s', (modelStr, expectedModel, expectedVoice) => {
+    const [model, voice] = parseTTSModelString(modelStr);
+    expect(model).toBe(expectedModel);
+    expect(voice).toBe(expectedVoice);
+  });
+
+  it('empty voice after colon', () => {
+    const [model, voice] = parseTTSModelString('cartesia/sonic:');
+    expect(model).toBe('cartesia/sonic');
+    expect(voice).toBe('');
+  });
+});
+
+describe('normalizeTTSFallback', () => {
+  it('single string model', () => {
+    const result = normalizeTTSFallback('cartesia/sonic');
+    expect(result).toEqual([{ model: 'cartesia/sonic', voice: '' }]);
+  });
+
+  it('single string model with voice', () => {
+    const result = normalizeTTSFallback('cartesia/sonic:my-voice');
+    expect(result).toEqual([{ model: 'cartesia/sonic', voice: 'my-voice' }]);
+  });
+
+  it('single FallbackModel dict', () => {
+    const fallback: TTSFallbackModel = { model: 'cartesia/sonic', voice: 'narrator' };
+    const result = normalizeTTSFallback(fallback);
+    expect(result).toEqual([{ model: 'cartesia/sonic', voice: 'narrator' }]);
+  });
+
+  it('list of string models', () => {
+    const result = normalizeTTSFallback(['cartesia/sonic', 'elevenlabs/eleven_flash_v2']);
+    expect(result).toEqual([
+      { model: 'cartesia/sonic', voice: '' },
+      { model: 'elevenlabs/eleven_flash_v2', voice: '' },
+    ]);
+  });
+
+  it('list of string models with voices', () => {
+    const result = normalizeTTSFallback(['cartesia/sonic:voice1', 'elevenlabs:voice2']);
+    expect(result).toEqual([
+      { model: 'cartesia/sonic', voice: 'voice1' },
+      { model: 'elevenlabs', voice: 'voice2' },
+    ]);
+  });
+
+  it('list of FallbackModel dicts', () => {
+    const fallbacks: TTSFallbackModel[] = [
+      { model: 'cartesia/sonic', voice: 'narrator' },
+      { model: 'elevenlabs', voice: '' },
+    ];
+    const result = normalizeTTSFallback(fallbacks);
+    expect(result).toEqual([
+      { model: 'cartesia/sonic', voice: 'narrator' },
+      { model: 'elevenlabs', voice: '' },
+    ]);
+  });
+
+  it('mixed list of strings and dicts', () => {
+    const result = normalizeTTSFallback([
+      'cartesia/sonic:voice1',
+      { model: 'elevenlabs/eleven_flash_v2', voice: 'custom' } as TTSFallbackModel,
+      'rime/mist',
+    ]);
+    expect(result).toEqual([
+      { model: 'cartesia/sonic', voice: 'voice1' },
+      { model: 'elevenlabs/eleven_flash_v2', voice: 'custom' },
+      { model: 'rime/mist', voice: '' },
+    ]);
+  });
+
+  it('FallbackModel with extraKwargs is preserved', () => {
+    const fallback: TTSFallbackModel = {
+      model: 'cartesia/sonic',
+      voice: 'narrator',
+      extraKwargs: { duration: 30.0, speed: 'fast' },
+    };
+    const result = normalizeTTSFallback(fallback);
+    expect(result).toEqual([
+      {
+        model: 'cartesia/sonic',
+        voice: 'narrator',
+        extraKwargs: { duration: 30.0, speed: 'fast' },
+      },
+    ]);
+  });
+
+  it('list with extraKwargs preserved', () => {
+    const result = normalizeTTSFallback([
+      { model: 'cartesia/sonic', voice: 'v1', extraKwargs: { speed: 'slow' } } as TTSFallbackModel,
+      'elevenlabs:voice2',
+      { model: 'rime/mist', voice: '', extraKwargs: { custom: true } } as TTSFallbackModel,
+    ]);
+    expect(result).toEqual([
+      { model: 'cartesia/sonic', voice: 'v1', extraKwargs: { speed: 'slow' } },
+      { model: 'elevenlabs', voice: 'voice2' },
+      { model: 'rime/mist', voice: '', extraKwargs: { custom: true } },
+    ]);
+  });
+
+  it('empty list returns empty list', () => {
+    const result = normalizeTTSFallback([]);
+    expect(result).toEqual([]);
+  });
+
+  it('FallbackModel with empty voice', () => {
+    const fallback: TTSFallbackModel = { model: 'cartesia/sonic', voice: '' };
+    const result = normalizeTTSFallback(fallback);
+    expect(result).toEqual([{ model: 'cartesia/sonic', voice: '' }]);
+  });
+});
+
+describe('TTS constructor fallback and connOptions', () => {
+  it('fallback not given defaults to undefined', () => {
+    const tts = makeTts();
+    expect(tts['opts'].fallback).toBeUndefined();
+  });
+
+  it('fallback single string is normalized', () => {
+    const tts = makeTts({ fallback: 'elevenlabs/eleven_flash_v2' });
+    expect(tts['opts'].fallback).toEqual([{ model: 'elevenlabs/eleven_flash_v2', voice: '' }]);
+  });
+
+  it('fallback single string with voice is normalized', () => {
+    const tts = makeTts({ fallback: 'cartesia/sonic:my-voice' });
+    expect(tts['opts'].fallback).toEqual([{ model: 'cartesia/sonic', voice: 'my-voice' }]);
+  });
+
+  it('fallback list of strings is normalized', () => {
+    const tts = makeTts({ fallback: ['cartesia/sonic', 'elevenlabs'] });
+    expect(tts['opts'].fallback).toEqual([
+      { model: 'cartesia/sonic', voice: '' },
+      { model: 'elevenlabs', voice: '' },
+    ]);
+  });
+
+  it('fallback single FallbackModel is normalized to list', () => {
+    const tts = makeTts({ fallback: { model: 'cartesia/sonic', voice: 'narrator' } });
+    expect(tts['opts'].fallback).toEqual([{ model: 'cartesia/sonic', voice: 'narrator' }]);
+  });
+
+  it('fallback with extraKwargs is preserved', () => {
+    const tts = makeTts({
+      fallback: {
+        model: 'cartesia/sonic',
+        voice: 'narrator',
+        extraKwargs: { duration: 30.0, speed: 'fast' },
+      },
+    });
+    expect(tts['opts'].fallback).toEqual([
+      {
+        model: 'cartesia/sonic',
+        voice: 'narrator',
+        extraKwargs: { duration: 30.0, speed: 'fast' },
+      },
+    ]);
+  });
+
+  it('fallback mixed list is normalized', () => {
+    const tts = makeTts({
+      fallback: [
+        'cartesia/sonic:voice1',
+        { model: 'elevenlabs', voice: 'custom', extraKwargs: { speed: 'slow' } },
+        'rime/mist',
+      ],
+    });
+    expect(tts['opts'].fallback).toEqual([
+      { model: 'cartesia/sonic', voice: 'voice1' },
+      { model: 'elevenlabs', voice: 'custom', extraKwargs: { speed: 'slow' } },
+      { model: 'rime/mist', voice: '' },
+    ]);
+  });
+
+  it('connOptions not given uses default', () => {
+    const tts = makeTts();
+    expect(tts['opts'].connOptions).toEqual(DEFAULT_API_CONNECT_OPTIONS);
+  });
+
+  it('connOptions custom timeout', () => {
+    const custom: APIConnectOptions = { timeoutMs: 30000, maxRetry: 3, retryIntervalMs: 2000 };
+    const tts = makeTts({ connOptions: custom });
+    expect(tts['opts'].connOptions).toEqual(custom);
+    expect(tts['opts'].connOptions!.timeoutMs).toBe(30000);
+  });
+
+  it('connOptions custom maxRetry', () => {
+    const custom: APIConnectOptions = { timeoutMs: 10000, maxRetry: 5, retryIntervalMs: 2000 };
+    const tts = makeTts({ connOptions: custom });
+    expect(tts['opts'].connOptions).toEqual(custom);
+    expect(tts['opts'].connOptions!.maxRetry).toBe(5);
+  });
+
+  it('connOptions full custom', () => {
+    const custom: APIConnectOptions = { timeoutMs: 60000, maxRetry: 10, retryIntervalMs: 2000 };
+    const tts = makeTts({ connOptions: custom });
+    expect(tts['opts'].connOptions).toEqual(custom);
+    expect(tts['opts'].connOptions!.timeoutMs).toBe(60000);
+    expect(tts['opts'].connOptions!.maxRetry).toBe(10);
+    expect(tts['opts'].connOptions!.retryIntervalMs).toBe(2000);
+  });
+});

package/src/inference/tts.ts

@@ -16,7 +16,6 @@ import { Event, Future, Task, cancelAndWait, combineSignals, shortuuid } from '.
 import {
   type TtsClientEvent,
   type TtsServerEvent,
-  type TtsSessionCreateEvent,
   ttsClientEventSchema,
   ttsServerEventSchema,
 } from './api_protos.js';
@@ -46,13 +45,17 @@ export type InworldModels =
 export type RimeModels = 'rime/arcana' | 'rime/mistv2';
 
 export interface CartesiaOptions {
-  duration?: number; // max duration of audio in seconds
-  speed?: 'slow' | 'normal' | 'fast'; // default: not specified
+  /** Maximum duration of audio in seconds. */
+  duration?: number;
+  /** Speech speed. Default: not specified. */
+  speed?: 'slow' | 'normal' | 'fast';
 }
 
 export interface ElevenlabsOptions {
-  inactivity_timeout?: number; // default: 60
-  apply_text_normalization?: 'auto' | 'off' | 'on'; // default: "auto"
+  /** Inactivity timeout in seconds. Default: 60. */
+  inactivity_timeout?: number;
+  /** Text normalization mode. Default: "auto". */
+  apply_text_normalization?: 'auto' | 'off' | 'on';
 }
 
 export interface DeepgramTTSOptions {}
@@ -90,6 +93,45 @@ export type TTSOptions<TModel extends TTSModels> = TModel extends CartesiaModels
           ? InworldOptions
           : Record<string, unknown>;
 
+/** Parse a model string into [model, voice]. Voice is undefined if not specified. */
+export function parseTTSModelString(model: string): [string, string | undefined] {
+  const idx = model.lastIndexOf(':');
+  if (idx !== -1) {
+    return [model.slice(0, idx), model.slice(idx + 1)];
+  }
+  return [model, undefined];
+}
+
+/** A fallback model with optional extra configuration. Extra fields are passed through to the provider. */
+export interface TTSFallbackModel {
+  /** Model name (e.g. "cartesia/sonic", "elevenlabs/eleven_flash_v2", "rime/arcana"). */
+  model: string;
+  /** Voice to use for the model. */
+  voice: string;
+  /** Extra configuration for the model. */
+  extraKwargs?: Record<string, unknown>;
+}
+
+export type TTSFallbackModelType = TTSFallbackModel | string;
+
+/** Normalize a single or list of FallbackModelType into TTSFallbackModel[]. */
+export function normalizeTTSFallback(
+  fallback: TTSFallbackModelType | TTSFallbackModelType[],
+): TTSFallbackModel[] {
+  const makeFallback = (model: TTSFallbackModelType): TTSFallbackModel => {
+    if (typeof model === 'string') {
+      const [name, voice] = parseTTSModelString(model);
+      return { model: name, voice: voice ?? '' };
+    }
+    return model;
+  };
+
+  if (Array.isArray(fallback)) {
+    return fallback.map(makeFallback);
+  }
+  return [makeFallback(fallback)];
+}
+
 type TTSEncoding = 'pcm_s16le';
 
 const DEFAULT_ENCODING: TTSEncoding = 'pcm_s16le';
@@ -108,6 +150,8 @@ export interface InferenceTTSOptions<TModel extends TTSModels> {
   apiKey: string;
   apiSecret: string;
   modelOptions: TTSOptions<TModel>;
+  fallback?: TTSFallbackModel[];
+  connOptions?: APIConnectOptions;
 }
 
 /**
@@ -130,6 +174,8 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
     apiKey?: string;
     apiSecret?: string;
     modelOptions?: TTSOptions<TModel>;
+    fallback?: TTSFallbackModelType | TTSFallbackModelType[];
+    connOptions?: APIConnectOptions;
   }) {
     const sampleRate = opts?.sampleRate ?? DEFAULT_SAMPLE_RATE;
     super(sampleRate, 1, { streaming: true });
@@ -143,6 +189,8 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
       apiKey,
       apiSecret,
       modelOptions = {} as TTSOptions<TModel>,
+      fallback,
+      connOptions,
     } = opts || {};
 
     const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
@@ -176,6 +224,8 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
       }
     }
 
+    const normalizedFallback = fallback ? normalizeTTSFallback(fallback) : undefined;
+
     this.opts = {
       model: nextModel,
       voice: nextVoice,
@@ -186,6 +236,8 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
       apiKey: lkApiKey,
       apiSecret: lkApiSecret,
       modelOptions,
+      fallback: normalizedFallback,
+      connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS,
     };
 
     // Initialize connection pool
@@ -203,11 +255,8 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
   }
 
   static fromModelString(modelString: string): TTS<AnyString> {
-    if (modelString.includes(':')) {
-      const [model, voice] = modelString.split(':') as [TTSModels, string];
-      return new TTS({ model, voice });
-    }
-    return new TTS({ model: modelString });
+    const [model, voice] = parseTTSModelString(modelString);
+    return new TTS({ model, voice: voice || undefined });
   }
 
   updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {
@@ -222,7 +271,7 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
   }
 
   stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream<TModel> {
-    const { connOptions = DEFAULT_API_CONNECT_OPTIONS } = options || {};
+    const { connOptions = this.opts.connOptions ?? DEFAULT_API_CONNECT_OPTIONS } = options || {};
     const stream = new SynthesizeStream(this, { ...this.opts }, connOptions);
     this.streams.add(stream);
     return stream;
@@ -243,11 +292,28 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
       sample_rate: String(this.opts.sampleRate),
       encoding: this.opts.encoding,
       extra: this.opts.modelOptions,
-    } as TtsSessionCreateEvent;
+    } as Record<string, unknown>;
+
+    if (this.opts.voice) (params as Record<string, unknown>).voice = this.opts.voice;
+    if (this.opts.model) (params as Record<string, unknown>).model = this.opts.model;
+    if (this.opts.language) (params as Record<string, unknown>).language = this.opts.language;
+
+    if (this.opts.fallback?.length) {
+      params.fallback = {
+        models: this.opts.fallback.map((m) => ({
+          model: m.model,
+          voice: m.voice,
+          extra: m.extraKwargs ?? {},
+        })),
+      };
+    }
 
-    if (this.opts.voice) params.voice = this.opts.voice;
-    if (this.opts.model) params.model = this.opts.model;
-    if (this.opts.language) params.language = this.opts.language;
+    if (this.opts.connOptions) {
+      params.connection = {
+        timeout: this.opts.connOptions.timeoutMs / 1000,
+        retries: this.opts.connOptions.maxRetry,
+      };
+    }
 
     this.#logger.debug({ url }, 'inference.TTS creating new websocket connection (pool miss)');
     const socket = await connectWs(url, headers, timeout);

package/src/ipc/inference_proc_lazy_main.ts

@@ -36,7 +36,19 @@ const ORPHANED_TIMEOUT = 15 * 1000;
 
     const runners: { [id: string]: InferenceRunner } = await Promise.all(
       Object.entries(JSON.parse(process.argv[2]!)).map(async ([k, v]) => {
-        return [k, await import(v as string).then((m) => new m.default())];
+        return [
+          k,
+          await import(v as string).then((m) => {
+            // Handle both ESM (m.default is the class) and CJS (m.default.default is the class)
+            const Runner = typeof m.default === 'function' ? m.default : m.default?.default;
+            if (typeof Runner !== 'function') {
+              throw new Error(
+                `Unable to load inference runner: Missing or invalid default export in ${v}`,
+              );
+            }
+            return new Runner();
+          }),
+        ];
       }),
     ).then(Object.fromEntries);
 

package/src/ipc/job_proc_lazy_main.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { Room, RoomEvent } from '@livekit/rtc-node';
+import { Room, RoomEvent, dispose } from '@livekit/rtc-node';
 import { EventEmitter, once } from 'node:events';
 import { pathToFileURL } from 'node:url';
 import type { Logger } from 'pino';
@@ -156,7 +156,11 @@ const startJob = (
     //   [2] import.meta.filename of function containing entry file
     const moduleFile = process.argv[2];
     const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {
-      const agent = module.default;
+      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)
+      const agent =
+        typeof module.default === 'function' || isAgent(module.default)
+          ? module.default
+          : module.default?.default;
       if (agent === undefined || !isAgent(agent)) {
         throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);
       }
@@ -241,6 +245,18 @@ const startJob = (
 
     await join.await;
 
+    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)
+    // before process.exit() to prevent libc++abi mutex crash during teardown.
+    // Without this, process.exit() can kill the process while native threads are
+    // still running, causing: "mutex lock failed: Invalid argument"
+    // See: https://github.com/livekit/node-sdks/issues/564
+    try {
+      await dispose();
+      logger.debug('native resources disposed');
+    } catch (error) {
+      logger.warn({ error }, 'failed to dispose native resources');
+    }
+
     logger.debug('Job process shutdown');
     process.exit(0);
   }

package/src/ipc/supervised_proc.ts

@@ -10,12 +10,19 @@ import { Future } from '../utils.js';
 import type { IPCMessage } from './message.js';
 
 export interface ProcOpts {
+  /** Timeout for process initialization in milliseconds. */
   initializeTimeout: number;
+  /** Timeout for process shutdown in milliseconds. */
   closeTimeout: number;
+  /** Memory usage warning threshold in megabytes. */
   memoryWarnMB: number;
+  /** Memory usage limit in megabytes. */
   memoryLimitMB: number;
+  /** Interval for health check pings in milliseconds. */
   pingInterval: number;
+  /** Timeout waiting for pong response in milliseconds. */
   pingTimeout: number;
+  /** Threshold for warning about unresponsive processes in milliseconds. */
   highPingThreshold: number;
 }
 

package/src/stt/stt.ts

@@ -49,15 +49,22 @@ export enum SpeechEventType {
 
 /** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
+  /** Language code of the speech. */
   language: string;
+  /** Transcribed text. */
   text: string;
+  /** Start time of the speech segment in seconds. */
   startTime: number;
+  /** End time of the speech segment in seconds. */
   endTime: number;
+  /** Confidence score of the transcription (0-1). */
   confidence: number;
+  /** Word-level timing information. */
   words?: TimedString[];
 }
 
 export interface RecognitionUsage {
+  /** Duration of the audio that was recognized in seconds. */
   audioDuration: number;
 }
 
@@ -344,6 +351,11 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
       }
     }
 
+    if (frame.samplesPerChannel === 0) {
+      this.input.put(frame);
+      return;
+    }
+
     if (this.resampler) {
       const frames = this.resampler.push(frame);
       for (const frame of frames) {

package/src/transcription.ts

@@ -13,11 +13,17 @@ import { AsyncIterableQueue, Future, shortuuid } from './utils.js';
 const STANDARD_SPEECH_RATE = 3830;
 
 export interface TextSyncOptions {
+  /** Language code for transcription. */
   language: string;
+  /** Speech speed multiplier. */
   speed: number;
+  /** Delay between sentences in milliseconds. */
   newSentenceDelay: number;
+  /** Tokenizer for splitting text into sentences. */
   sentenceTokenizer: SentenceTokenizer;
+  /** Function to hyphenate words. */
   hyphenateWord: (word: string) => string[];
+  /** Function to split text into words with positions. */
   splitWords: (words: string) => [string, number, number][];
 }
 

package/src/utils.ts

@@ -651,14 +651,22 @@ export function resampleStream({
   let resampler: AudioResampler | null = null;
   const transformStream = new TransformStream<AudioFrame, AudioFrame>({
     transform(chunk: AudioFrame, controller: TransformStreamDefaultController<AudioFrame>) {
+      if (chunk.samplesPerChannel === 0) {
+        controller.enqueue(chunk);
+        return;
+      }
       if (!resampler) {
         resampler = new AudioResampler(chunk.sampleRate, outputRate);
       }
       for (const frame of resampler.push(chunk)) {
         controller.enqueue(frame);
       }
-      for (const frame of resampler.flush()) {
-        controller.enqueue(frame);
+    },
+    flush(controller) {
+      if (resampler) {
+        for (const frame of resampler.flush()) {
+          controller.enqueue(frame);
+        }
       }
     },
   });

package/src/vad.ts

@@ -30,9 +30,9 @@ export interface VADEvent {
   samplesIndex: number;
   /** Timestamp when the event was fired. */
   timestamp: number;
-  /** Duration of the speech segment. */
+  /** Duration of the speech segment in seconds. */
   speechDuration: number;
-  /** Duration of the silence segment. */
+  /** Duration of the silence segment in seconds. */
   silenceDuration: number;
   /**
    * List of audio frames associated with the speech.
@@ -56,6 +56,7 @@ export interface VADEvent {
 }
 
 export interface VADCapabilities {
+  /** Duration of each VAD inference window in milliseconds. Used to batch metrics emissions to roughly once per second. */
   updateInterval: number;
 }
 
@@ -154,7 +155,7 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
       switch (value.type) {
         case VADEventType.START_OF_SPEECH:
           inferenceCount++;
-          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {
+          if (inferenceCount >= 1000 / this.#vad.capabilities.updateInterval) {
             this.#vad.emit('metrics_collected', {
               type: 'vad_metrics',
               timestamp: Date.now(),

package/src/voice/agent_activity.ts

@@ -1023,7 +1023,7 @@ export class AgentActivity implements RecognitionHooks {
               toolChoice: toOaiToolChoice(toolChoice !== undefined ? toolChoice : this.toolChoice),
             },
             abortController,
-            instructions ? `${this.agent.instructions}\n${instructions}` : instructions,
+            instructions,
             userMessage,
           ),
         ),

package/src/voice/audio_recognition.ts

@@ -18,11 +18,17 @@ import type { TurnDetectionMode } from './agent_session.js';
 import type { STTNode } from './io.js';
 
 export interface EndOfTurnInfo {
+  /** The new transcript text from the user's speech. */
   newTranscript: string;
+  /** Confidence score of the transcript (0-1). */
   transcriptConfidence: number;
+  /** Delay from speech stop to final transcription in milliseconds. */
   transcriptionDelay: number;
+  /** Delay from speech stop to end of utterance detection in milliseconds. */
   endOfUtteranceDelay: number;
+  /** Timestamp when user started speaking (milliseconds since epoch). */
   startedSpeakingAt: number | undefined;
+  /** Timestamp when user stopped speaking (milliseconds since epoch). */
   stoppedSpeakingAt: number | undefined;
 }
 
@@ -50,13 +56,21 @@ export interface _TurnDetector {
 }
 
 export interface AudioRecognitionOptions {
+  /** Hooks for recognition events. */
   recognitionHooks: RecognitionHooks;
+  /** Speech-to-text node. */
   stt?: STTNode;
+  /** Voice activity detection. */
   vad?: VAD;
+  /** Turn detector for end-of-turn prediction. */
   turnDetector?: _TurnDetector;
+  /** Turn detection mode. */
   turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
+  /** Minimum endpointing delay in milliseconds. */
   minEndpointingDelay: number;
+  /** Maximum endpointing delay in milliseconds. */
   maxEndpointingDelay: number;
+  /** Root span context for tracing. */
   rootSpanContext?: Context;
 }