@livekit/agents 1.2.0 → 1.2.2

package/src/inference/interruption/ws_transport.ts

@@ -1,11 +1,13 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { Throws } from '@livekit/throws-transformer/throws';
 import { TransformStream } from 'stream/web';
 import WebSocket from 'ws';
 import { z } from 'zod';
 import { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';
 import { log } from '../../log.js';
+import TypedPromise from '../../typed_promise.js';
 import { createAccessToken } from '../utils.js';
 import { InterruptionCacheEntry } from './interruption_cache_entry.js';
 import type { OverlappingSpeechEvent } from './types.js';
@@ -70,7 +72,9 @@ type WsMessage = z.infer<typeof wsMessageSchema>;
 /**
  * Creates a WebSocket connection and waits for it to open.
  */
-async function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {
+async function connectWebSocket(
+  options: WsTransportOptions,
+): Promise<Throws<WebSocket, APIStatusError | APITimeoutError | APIConnectionError>> {
   const baseUrl = options.baseUrl.replace(/^http/, 'ws');
   const token = await createAccessToken(options.apiKey, options.apiSecret);
   const url = `${baseUrl}/bargein`;
@@ -79,37 +83,39 @@ async function connectWebSocket(options: WsTransportOptions): Promise<WebSocket>
     headers: { Authorization: `Bearer ${token}` },
   });
 
-  await new Promise<void>((resolve, reject) => {
-    const timeout = setTimeout(() => {
-      ws.terminate();
-      reject(
-        new APITimeoutError({
-          message: 'WebSocket connection timeout',
-          options: { retryable: false },
-        }),
-      );
-    }, options.timeout);
-    ws.once('open', () => {
-      clearTimeout(timeout);
-      resolve();
-    });
-    ws.once('unexpected-response', (_req, res) => {
-      clearTimeout(timeout);
-      ws.terminate();
-      const statusCode = res.statusCode ?? -1;
-      reject(
-        new APIStatusError({
-          message: `WebSocket connection rejected with status ${statusCode}`,
-          options: { statusCode, retryable: false },
-        }),
-      );
-    });
-    ws.once('error', (err: Error) => {
-      clearTimeout(timeout);
-      ws.terminate();
-      reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
-    });
-  });
+  await new TypedPromise<void, APIStatusError | APITimeoutError | APIConnectionError>(
+    (resolve, reject) => {
+      const timeout = setTimeout(() => {
+        ws.terminate();
+        reject(
+          new APITimeoutError({
+            message: 'WebSocket connection timeout',
+            options: { retryable: false },
+          }),
+        );
+      }, options.timeout);
+      ws.once('open', () => {
+        clearTimeout(timeout);
+        resolve();
+      });
+      ws.once('unexpected-response', (_req, res) => {
+        clearTimeout(timeout);
+        ws.terminate();
+        const statusCode = res.statusCode ?? -1;
+        reject(
+          new APIStatusError({
+            message: `WebSocket connection rejected with status ${statusCode}`,
+            options: { statusCode, retryable: false },
+          }),
+        );
+      });
+      ws.once('error', (err: Error) => {
+        clearTimeout(timeout);
+        ws.terminate();
+        reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
+      });
+    },
+  );
 
   return ws;
 }
@@ -159,7 +165,9 @@ export function createWsTransport(
     });
   }
 
-  async function ensureConnection(): Promise<void> {
+  async function ensureConnection(): Promise<
+    Throws<void, APIStatusError | APITimeoutError | APIConnectionError>
+  > {
     if (ws && ws.readyState === WebSocket.OPEN) return;
 
     ws = await connectWebSocket(options);
@@ -357,7 +365,9 @@ export function createWsTransport(
     {
       async start(controller) {
         outputController = controller;
-        await ensureConnection();
+        await ensureConnection().catch((e) => {
+          controller.error(e);
+        });
       },
 
       transform(chunk, controller) {

package/src/inference/tts.test.ts

@@ -265,3 +265,90 @@ describe('TTS constructor fallback and connOptions', () => {
     expect(tts['opts'].connOptions!.retryIntervalMs).toBe(2000);
   });
 });
+
+describe('TTS provider modelOptions parity', () => {
+  it('preserves ElevenLabs inference model options', () => {
+    const modelOptions = {
+      speed: 1.2,
+      stability: 0.5,
+      similarity_boost: 0.8,
+      enable_logging: false,
+    };
+
+    const tts = new TTS({
+      model: 'elevenlabs/eleven_flash_v2_5' as const,
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseURL: 'https://example.livekit.cloud',
+      modelOptions,
+    });
+
+    expect(tts['opts'].modelOptions).toEqual(modelOptions);
+  });
+
+  it('accepts expanded Cartesia inference model options', () => {
+    const modelOptions = {
+      speed: 1.15,
+      emotion: 'curious',
+      add_timestamps: true,
+    };
+
+    const tts = new TTS({
+      model: 'cartesia/sonic' as const,
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseURL: 'https://example.livekit.cloud',
+      modelOptions,
+    });
+
+    expect(tts['opts'].modelOptions).toEqual(modelOptions);
+  });
+
+  it('accepts Deepgram inference model options', () => {
+    const modelOptions = { mip_opt_out: true };
+
+    const tts = new TTS({
+      model: 'deepgram/aura-2' as const,
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseURL: 'https://example.livekit.cloud',
+      modelOptions,
+    });
+
+    expect(tts['opts'].modelOptions).toEqual(modelOptions);
+  });
+
+  it('accepts Rime inference model options', () => {
+    const modelOptions = {
+      speed_alpha: 0.9,
+      pause_between_brackets: true,
+    };
+
+    const tts = new TTS({
+      model: 'rime/mistv2' as const,
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseURL: 'https://example.livekit.cloud',
+      modelOptions,
+    });
+
+    expect(tts['opts'].modelOptions).toEqual(modelOptions);
+  });
+
+  it('accepts Inworld inference model options', () => {
+    const modelOptions = {
+      timestamp_type: 'WORD' as const,
+      apply_text_normalization: 'ON' as const,
+    };
+
+    const tts = new TTS({
+      model: 'inworld/inworld-tts-1' as const,
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseURL: 'https://example.livekit.cloud',
+      modelOptions,
+    });
+
+    expect(tts['opts'].modelOptions).toEqual(modelOptions);
+  });
+});

package/src/inference/tts.ts

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import { WebSocket } from 'ws';
-import { APIError, APIStatusError } from '../_exceptions.js';
+import { APIError, APIStatusError, APITimeoutError } from '../_exceptions.js';
 import { AudioByteStream } from '../audio.js';
 import { ConnectionPool } from '../connection_pool.js';
 import { type LanguageCode, normalizeLanguage } from '../language.js';
@@ -13,7 +13,15 @@ import { basic as tokenizeBasic } from '../tokenize/index.js';
 import type { ChunkedStream } from '../tts/index.js';
 import { SynthesizeStream as BaseSynthesizeStream, TTS as BaseTTS } from '../tts/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
-import { Event, Future, Task, cancelAndWait, combineSignals, shortuuid } from '../utils.js';
+import {
+  Event,
+  Future,
+  Task,
+  cancelAndWait,
+  combineSignals,
+  shortuuid,
+  waitUntilTimeout,
+} from '../utils.js';
 import {
   type TtsClientEvent,
   type TtsServerEvent,
@@ -46,10 +54,16 @@ export type InworldModels =
 export type RimeModels = 'rime/arcana' | 'rime/mistv2';
 
 export interface CartesiaOptions {
+  emotion?: string;
   /** Maximum duration of audio in seconds. */
   duration?: number;
   /** Speech speed. Default: not specified. */
-  speed?: 'slow' | 'normal' | 'fast';
+  speed?: 'slow' | 'normal' | 'fast' | number;
+  volume?: number;
+  max_buffer_delay_ms?: number;
+  add_timestamps?: boolean;
+  add_phoneme_timestamps?: boolean;
+  use_normalized_timestamps?: boolean;
 }
 
 export interface ElevenlabsOptions {
@@ -57,18 +71,50 @@ export interface ElevenlabsOptions {
   inactivity_timeout?: number;
   /** Text normalization mode. Default: "auto". */
   apply_text_normalization?: 'auto' | 'off' | 'on';
+  auto_mode?: boolean;
+  enable_logging?: boolean;
+  enable_ssml_parsing?: boolean;
+  sync_alignment?: boolean;
+  language_code?: string;
+  /** Voice stability tuning, typically in the range [0, 1]. */
+  stability?: number;
+  /** Voice similarity tuning, typically in the range [0, 1]. */
+  similarity_boost?: number;
+  /** Style exaggeration tuning, typically in the range [0, 1]. */
+  style?: number;
+  /** Speech speed multiplier. */
+  speed?: number;
+  use_speaker_boost?: boolean;
+  chunk_length_schedule?: number[];
+  preferred_alignment?: string;
 }
 
-export interface DeepgramTTSOptions {}
+export interface DeepgramTTSOptions {
+  /** Default: false. */
+  mip_opt_out?: boolean;
+}
 
-export interface RimeOptions {}
+export interface RimeOptions {
+  /** Default 1.0, <1 = faster, >1 = slower. */
+  speed_alpha?: number;
+  /** Default false. */
+  pause_between_brackets?: boolean;
+  /** Default false. */
+  phonemize_between_brackets?: boolean;
+  /** Comma-separated speed factors for [bracketed] words. */
+  inline_speed_alpha?: string;
+  /** Default false. */
+  no_text_normalization?: boolean;
+}
 
 export interface InworldOptions {
-  /** Controls how fast the voice speaks. 1.0 is normal speed, 0.5 is half, 1.5 is 1.5x. Default: 1.0. */
+  /** Range >0.5, <=1.5. */
   speaking_rate?: number;
-  /** Controls randomness in the output. Recommended between 0.6 and 1.1. Default: 1.1. */
+  /** Range 0-2. */
   temperature?: number;
-  /** Controls text normalization. "ON" expands numbers, dates, abbreviations. "OFF" reads text as written. Default: "ON". */
+  timestamp_type?: 'TIMESTAMP_TYPE_UNSPECIFIED' | 'WORD' | 'CHARACTER';
+  apply_text_normalization?: 'APPLY_TEXT_NORMALIZATION_UNSPECIFIED' | 'ON' | 'OFF';
+  /** @deprecated Backward-compatible alias. Use `apply_text_normalization`. */
   text_normalization?: 'ON' | 'OFF';
 }
 
@@ -156,6 +202,7 @@ export interface InferenceTTSOptions<TModel extends TTSModels> {
   baseURL: string;
   apiKey: string;
   apiSecret: string;
+  /** Flat provider-specific inference options forwarded as the `extra` payload field. */
   modelOptions: TTSOptions<TModel>;
   fallback?: TTSFallbackModel[];
   connOptions?: APIConnectOptions;
@@ -180,6 +227,7 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
     sampleRate?: number;
     apiKey?: string;
     apiSecret?: string;
+    /** Flat provider-specific inference options forwarded as the `extra` payload field. */
     modelOptions?: TTSOptions<TModel>;
     fallback?: TTSFallbackModelType | TTSFallbackModelType[];
     connOptions?: APIConnectOptions;
@@ -538,6 +586,7 @@ export class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeSt
 
     const createRecvTask = async (signal: AbortSignal) => {
       let currentSessionId: string | null = null;
+      const recvTimeoutMs = this.connOptions.timeoutMs;
 
       const bstream = new AudioByteStream(this.opts.sampleRate, NUM_CHANNELS);
       const serverEventStream = eventChannel.stream();
@@ -547,7 +596,12 @@ export class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeSt
         await inputSentEvent.wait();
 
         while (!this.closed && !signal.aborted) {
-          const result = await reader.read();
+          const result = await waitUntilTimeout(
+            reader.read(),
+            recvTimeoutMs,
+            () => new APITimeoutError({ message: 'TTS recv idle timeout' }),
+          );
+
           if (signal.aborted) return;
           if (result.done) return;
 
@@ -592,6 +646,14 @@ export class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeSt
               break;
           }
         }
+      } catch (e) {
+        if (e instanceof APITimeoutError) {
+          this.#logger.warn('TTS recv task timed out waiting for server message');
+          await resourceCleanup();
+          completionFuture.reject(e);
+          return;
+        }
+        throw e;
       } finally {
         reader.releaseLock();
         try {

package/src/ipc/job_proc_lazy_main.ts

@@ -50,6 +50,7 @@ class PendingInference {
 
 class InfClient implements InferenceExecutor {
   #requests: { [id: string]: PendingInference } = {};
+  #logger = log();
 
   constructor() {
     process.on('message', (msg: IPCMessage) => {
@@ -58,7 +59,7 @@ class InfClient implements InferenceExecutor {
           const fut = this.#requests[msg.value.requestId];
           delete this.#requests[msg.value.requestId];
           if (!fut) {
-            log().child({ resp: msg.value }).warn('received unexpected inference response');
+            this.#logger.child({ resp: msg.value }).warn('received unexpected inference response');
             return;
           }
           fut.resolve(msg.value);
@@ -70,7 +71,11 @@ class InfClient implements InferenceExecutor {
   async doInference(method: string, data: unknown): Promise<unknown> {
     const requestId = shortuuid('inference_job_');
     if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {
-      throw new Error('IPC channel closed');
+      this.#logger.debug(
+        { method, requestId },
+        'IPC channel closed during inference, aborting gracefully',
+      );
+      throw new Error(`Inference ${method} aborted: IPC channel closed (expected during shutdown)`);
     }
 
     this.#requests[requestId] = new PendingInference();

package/src/ipc/supervised_proc.test.ts

@@ -125,6 +125,102 @@ describe('IPC send on dead process', () => {
   });
 });
 
+describe('init timeout rejection handling', () => {
+  it('does not produce unhandled rejection when init times out', async () => {
+    // Regression test: before the fix, run() was called without await in start().
+    // When init timed out, the rejection in run()'s `await this.init.await` escaped
+    // as an unhandled rejection — crashing the Node.js process.
+    const unhandled: unknown[] = [];
+    const handler = (reason: unknown) => unhandled.push(reason);
+    process.on('unhandledRejection', handler);
+
+    // Child that responds AFTER the timeout — simulates slow init under CPU pressure.
+    // Timeout fires at 50ms (init.reject), child responds at 200ms (once() resolves).
+    // Before the fix, init.reject caused an unhandled rejection in run().
+    const slowScript = join(tmpdir(), 'test_slow_init_child.mjs');
+    writeFileSync(
+      slowScript,
+      `process.on('message', () => {
+        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);
+      });
+      setInterval(() => {}, 1000);`,
+    );
+
+    const { SupervisedProc } = await import('./supervised_proc.js');
+    class TestProc extends SupervisedProc {
+      createProcess() {
+        return fork(slowScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });
+      }
+      async mainTask() {}
+    }
+
+    const proc = new TestProc(
+      50, // initializeTimeout — fires before child responds at 200ms
+      1000, // closeTimeout
+      0, // memoryWarnMB
+      0, // memoryLimitMB
+      5000, // pingInterval
+      60000, // pingTimeout
+      2500, // highPingThreshold
+    );
+
+    await proc.start();
+    // initialize() returns normally: child responds at 200ms, once() resolves,
+    // but init was already rejected at 50ms — run() gets the rejection.
+    await proc.initialize();
+
+    // Give the event loop a tick for any unhandled rejection to surface
+    await new Promise((r) => setTimeout(r, 100));
+
+    process.off('unhandledRejection', handler);
+    proc.proc?.kill();
+    try {
+      unlinkSync(slowScript);
+    } catch {}
+
+    expect(unhandled).toEqual([]);
+  });
+
+  it('join() resolves after init timeout instead of hanging forever', async () => {
+    // When run() fails early (before registering proc event handlers),
+    // #join must still resolve so that join() and close() don't hang.
+    const slowScript = join(tmpdir(), 'test_slow_init_child_join.mjs');
+    writeFileSync(
+      slowScript,
+      `process.on('message', () => {
+        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);
+      });
+      setInterval(() => {}, 1000);`,
+    );
+
+    const { SupervisedProc } = await import('./supervised_proc.js');
+    class TestProc extends SupervisedProc {
+      createProcess() {
+        return fork(slowScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });
+      }
+      async mainTask() {}
+    }
+
+    const proc = new TestProc(50, 1000, 0, 0, 5000, 60000, 2500);
+
+    await proc.start();
+    await proc.initialize();
+
+    // join() must resolve within a reasonable time, not hang forever
+    const result = await Promise.race([
+      proc.join().then(() => 'resolved'),
+      new Promise((r) => setTimeout(() => r('timeout'), 2000)),
+    ]);
+
+    proc.proc?.kill();
+    try {
+      unlinkSync(slowScript);
+    } catch {}
+
+    expect(result).toBe('resolved');
+  });
+});
+
 describe('timer cleanup', () => {
   it('clearInterval stops the interval', async () => {
     let count = 0;

package/src/ipc/supervised_proc.ts

@@ -84,7 +84,14 @@ export abstract class SupervisedProc {
     this.proc = this.createProcess();
 
     this.#started = true;
-    this.run();
+    this.run().catch((err) => {
+      this.#logger.child({ err }).warn('supervised process run failed');
+      // Note: we intentionally do NOT kill the child process here. Killing it
+      // would race with initialize()'s `once(proc, 'message')`, causing
+      // initialize() to hang forever and deadlocking the caller (proc_pool).
+      // The child process is cleaned up when the pool shuts down.
+      this.#join.resolve();
+    });
   }
 
   async run() {

package/src/job.ts

@@ -283,6 +283,7 @@ export class JobContext {
       startedAt: targetSession._startedAt,
       audioRecordingPath: recorderIO?.outputPath,
       audioRecordingStartedAt: recorderIO?.recordingStartedAt,
+      modelUsage: targetSession._usageCollector.flatten(),
     });
   }