@livekit/agents 1.0.23 → 1.0.25

package/src/stream/deferred_stream.ts

@@ -9,15 +9,22 @@ import type {
 import { IdentityTransform } from './identity_transform.js';
 
 /**
- * Check if error is related to reader.read after release lock
+ * Check if error is related to stream cleanup operations.
+ *
+ * These errors are expected when calling reader.read() after releaseLock()
+ * or when writing to already closed streams during cleanup:
  *
  * Invalid state: Releasing reader
  * Invalid state: The reader is not attached to a stream
+ * Invalid state: Controller is already closed
+ * Invalid state: WritableStream is closed
  */
 export function isStreamReaderReleaseError(e: unknown) {
   const allowedMessages = [
     'Invalid state: Releasing reader',
     'Invalid state: The reader is not attached to a stream',
+    'Controller is already closed',
+    'WritableStream is closed',
   ];
 
   if (e instanceof TypeError) {
@@ -66,18 +73,27 @@ export class DeferredReadableStream<T> {
         await this.writer.write(value);
       }
     } catch (e) {
-      // skip source detach related errors
+      // skip stream cleanup related errors
       if (isStreamReaderReleaseError(e)) return;
+
       sourceError = e;
     } finally {
       // any other error from source will be propagated to the consumer
       if (sourceError) {
-        this.writer.abort(sourceError);
+        try {
+          this.writer.abort(sourceError);
+        } catch (e) {
+          // ignore if writer is already closed
+        }
         return;
       }
 
       // release lock so this.stream.getReader().read() will terminate with done: true
-      this.writer.releaseLock();
+      try {
+        this.writer.releaseLock();
+      } catch (e) {
+        // ignore if writer lock is already released
+      }
 
       // we only close the writable stream after done
       try {
@@ -98,7 +114,8 @@ export class DeferredReadableStream<T> {
    */
   async detachSource() {
     if (!this.isSourceSet) {
-      throw new Error('Source not set');
+      // No-op if source was never set - this is a common case during cleanup
+      return;
     }
 
     // release lock will make any pending read() throw TypeError

package/src/stt/stream_adapter.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import { log } from '../log.js';
+import type { APIConnectOptions } from '../types.js';
 import type { VAD, VADStream } from '../vad.js';
 import { VADEventType } from '../vad.js';
 import type { SpeechEvent } from './stt.js';
@@ -22,14 +23,18 @@ export class StreamAdapter extends STT {
     this.#stt.on('metrics_collected', (metrics) => {
       this.emit('metrics_collected', metrics);
     });
+
+    this.#stt.on('error', (error) => {
+      this.emit('error', error);
+    });
   }
 
-  _recognize(frame: AudioFrame): Promise<SpeechEvent> {
-    return this.#stt.recognize(frame);
+  _recognize(frame: AudioFrame, abortSignal?: AbortSignal): Promise<SpeechEvent> {
+    return this.#stt.recognize(frame, abortSignal);
   }
 
-  stream(): StreamAdapterWrapper {
-    return new StreamAdapterWrapper(this.#stt, this.#vad);
+  stream(options?: { connOptions?: APIConnectOptions }): StreamAdapterWrapper {
+    return new StreamAdapterWrapper(this.#stt, this.#vad, options?.connOptions);
   }
 }
 
@@ -38,13 +43,18 @@ export class StreamAdapterWrapper extends SpeechStream {
   #vadStream: VADStream;
   label: string;
 
-  constructor(stt: STT, vad: VAD) {
-    super(stt);
+  constructor(stt: STT, vad: VAD, connOptions?: APIConnectOptions) {
+    super(stt, undefined, connOptions);
     this.#stt = stt;
     this.#vadStream = vad.stream();
     this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;
   }
 
+  close() {
+    super.close();
+    this.#vadStream.close();
+  }
+
   async monitorMetrics() {
     return; // do nothing
   }
@@ -71,7 +81,7 @@ export class StreamAdapterWrapper extends SpeechStream {
             this.output.put({ type: SpeechEventType.END_OF_SPEECH });
 
             try {
-              const event = await this.#stt.recognize(ev.frames);
+              const event = await this.#stt.recognize(ev.frames, this.abortSignal);
               if (!event.alternatives![0].text) {
                 continue;
               }
@@ -92,6 +102,6 @@ export class StreamAdapterWrapper extends SpeechStream {
       }
     };
 
-    Promise.all([forwardInput(), recognize()]);
+    await Promise.all([forwardInput(), recognize()]);
   }
 }

package/src/stt/stt.ts

@@ -10,7 +10,7 @@ import { calculateAudioDurationSeconds } from '../audio.js';
 import { log } from '../log.js';
 import type { STTMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
-import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
 
@@ -113,9 +113,9 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCal
   }
 
   /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
-  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {
+  async recognize(frame: AudioBuffer, abortSignal?: AbortSignal): Promise<SpeechEvent> {
     const startTime = process.hrtime.bigint();
-    const event = await this._recognize(frame);
+    const event = await this._recognize(frame, abortSignal);
     const durationMs = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));
     this.emit('metrics_collected', {
       type: 'stt_metrics',
@@ -128,13 +128,19 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCal
     });
     return event;
   }
-  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;
+
+  protected abstract _recognize(
+    frame: AudioBuffer,
+    abortSignal?: AbortSignal,
+  ): Promise<SpeechEvent>;
 
   /**
    * Returns a {@link SpeechStream} that can be used to push audio frames and receive
    * transcriptions
+   *
+   * @param options - Optional configuration including connection options
    */
-  abstract stream(): SpeechStream;
+  abstract stream(options?: { connOptions?: APIConnectOptions }): SpeechStream;
 
   async close(): Promise<void> {
     return;
@@ -171,6 +177,8 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
   private logger = log();
   private _connOptions: APIConnectOptions;
 
+  protected abortController = new AbortController();
+
   constructor(
     stt: STT,
     sampleRate?: number,
@@ -196,7 +204,7 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
         return await this.run();
       } catch (error) {
         if (error instanceof APIError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+          const retryInterval = intervalForRetry(this._connOptions, i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -288,6 +296,10 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
 
   protected abstract run(): Promise<void>;
 
+  protected get abortSignal(): AbortSignal {
+    return this.abortController.signal;
+  }
+
   updateInputStream(audioStream: ReadableStream<AudioFrame>) {
     this.deferredInputStream.setSource(audioStream);
   }
@@ -352,6 +364,7 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
     if (!this.input.closed) this.input.close();
     if (!this.queue.closed) this.queue.close();
     if (!this.output.closed) this.output.close();
+    if (!this.abortController.signal.aborted) this.abortController.abort();
     this.closed = true;
   }
 

package/src/telemetry/traces.ts

@@ -21,6 +21,7 @@ import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace
 import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
 import FormData from 'form-data';
 import { AccessToken } from 'livekit-server-sdk';
+import fs from 'node:fs/promises';
 import type { ChatContent, ChatItem } from '../llm/index.js';
 import { enableOtelLogging } from '../log.js';
 import type { SessionReport } from '../voice/report.js';
@@ -497,12 +498,13 @@ export async function uploadSessionReport(options: {
   const formData = new FormData();
 
   // Add header (protobuf MetricsRecordingHeader)
+  const audioStartTime = report.audioRecordingStartedAt ?? 0;
   const headerMsg = new MetricsRecordingHeader({
     roomId: report.roomId,
     duration: BigInt(0), // TODO: Calculate actual duration from report
     startTime: {
-      seconds: BigInt(Math.floor(report.timestamp / 1000)),
-      nanos: Math.floor((report.timestamp % 1000) * 1e6),
+      seconds: BigInt(Math.floor(audioStartTime / 1000)),
+      nanos: Math.floor((audioStartTime % 1000) * 1e6),
     },
   });
 
@@ -530,7 +532,27 @@ export async function uploadSessionReport(options: {
     },
   });
 
-  // TODO(brian): Add audio recording file when recorder IO is implemented
+  // Add audio recording file if available
+  if (report.audioRecordingPath && report.audioRecordingStartedAt) {
+    let audioBytes: Buffer;
+    try {
+      audioBytes = await fs.readFile(report.audioRecordingPath);
+    } catch {
+      audioBytes = Buffer.alloc(0);
+    }
+
+    if (audioBytes.length > 0) {
+      formData.append('audio', audioBytes, {
+        filename: 'recording.ogg',
+        contentType: 'audio/ogg',
+        knownLength: audioBytes.length,
+        header: {
+          'Content-Type': 'audio/ogg',
+          'Content-Length': audioBytes.length.toString(),
+        },
+      });
+    }
+  }
 
   // Upload to LiveKit Cloud using form-data's submit method
   // This properly streams the multipart form with all headers including Content-Length

package/src/tts/stream_adapter.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';
+import type { APIConnectOptions } from '../types.js';
 import { Task } from '../utils.js';
 import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
@@ -27,8 +28,8 @@ export class StreamAdapter extends TTS {
     return this.#tts.synthesize(text);
   }
 
-  stream(): StreamAdapterWrapper {
-    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);
+  stream(options?: { connOptions?: APIConnectOptions }): StreamAdapterWrapper {
+    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer, options?.connOptions);
   }
 }
 
@@ -37,8 +38,8 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   #sentenceStream: SentenceStream;
   label: string;
 
-  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
-    super(tts);
+  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer, connOptions?: APIConnectOptions) {
+    super(tts, connOptions);
     this.#tts = tts;
     this.#sentenceStream = sentenceTokenizer.stream();
     this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;

package/src/tts/tts.ts

@@ -11,7 +11,7 @@ import { log } from '../log.js';
 import type { TTSMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { recordException, traceTypes, tracer } from '../telemetry/index.js';
-import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';
 
 /** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
@@ -94,8 +94,10 @@ export abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCal
 
   /**
    * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
+   *
+   * @param options - Optional configuration including connection options
    */
-  abstract stream(): SynthesizeStream;
+  abstract stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream;
 
   async close(): Promise<void> {
     return;
@@ -186,7 +188,7 @@ export abstract class SynthesizeStream
         );
       } catch (error) {
         if (error instanceof APIError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+          const retryInterval = intervalForRetry(this._connOptions, i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -454,7 +456,7 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
         );
       } catch (error) {
         if (error instanceof APIError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+          const retryInterval = intervalForRetry(this._connOptions, i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });

package/src/types.ts

@@ -1,42 +1,66 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-export class APIConnectOptions {
-  /** Maximum number of retries to connect to the API. */
-  readonly maxRetry: number;
-  /** Interval between retries to connect to the API in milliseconds. */
-  readonly retryIntervalMs: number;
-  /** Timeout for connecting to the API in milliseconds. */
-  readonly timeoutMs: number;
 
-  constructor(options: Partial<APIConnectOptions> = {}) {
-    this.maxRetry = options.maxRetry ?? 3;
-    this.retryIntervalMs = options.retryIntervalMs ?? 2000;
-    this.timeoutMs = options.timeoutMs ?? 10000;
+/**
+ * Connection options for API calls, controlling retry and timeout behavior.
+ */
+export interface APIConnectOptions {
+  /** Maximum number of retries to connect to the API. Default: 3 */
+  maxRetry: number;
+  /** Interval between retries to connect to the API in milliseconds. Default: 2000 */
+  retryIntervalMs: number;
+  /** Timeout for connecting to the API in milliseconds. Default: 10000 */
+  timeoutMs: number;
+}
 
-    if (this.maxRetry < 0) {
-      throw new Error('maxRetry must be greater than or equal to 0');
-    }
-    if (this.retryIntervalMs < 0) {
-      throw new Error('retryIntervalMs must be greater than or equal to 0');
-    }
-    if (this.timeoutMs < 0) {
-      throw new Error('timeoutMs must be greater than or equal to 0');
-    }
-  }
+export const DEFAULT_API_CONNECT_OPTIONS: APIConnectOptions = {
+  maxRetry: 3,
+  retryIntervalMs: 2000,
+  timeoutMs: 10000,
+};
 
-  /** @internal */
-  _intervalForRetry(numRetries: number): number {
-    /**
-     * Return the interval for the given number of retries.
-     *
-     * The first retry is immediate, and then uses specified retryIntervalMs
-     */
-    if (numRetries === 0) {
-      return 0.1;
-    }
-    return this.retryIntervalMs;
+/**
+ * Return the interval for the given number of retries.
+ * The first retry is immediate, and then uses specified retryIntervalMs.
+ * @internal
+ */
+export function intervalForRetry(connOptions: APIConnectOptions, numRetries: number): number {
+  if (numRetries === 0) {
+    return 0.1;
   }
+  return connOptions.retryIntervalMs;
+}
+
+/**
+ * Connection options for the agent session, controlling retry and timeout behavior
+ * for STT, LLM, and TTS connections.
+ */
+export interface SessionConnectOptions {
+  /** Connection options for speech-to-text. */
+  sttConnOptions?: Partial<APIConnectOptions>;
+  /** Connection options for the language model. */
+  llmConnOptions?: Partial<APIConnectOptions>;
+  /** Connection options for text-to-speech. */
+  ttsConnOptions?: Partial<APIConnectOptions>;
+  /** Maximum number of consecutive unrecoverable errors from LLM or TTS before closing the session. Default: 3 */
+  maxUnrecoverableErrors?: number;
+}
+
+/**
+ * Resolved session connect options with all values populated.
+ * @internal
+ */
+export interface ResolvedSessionConnectOptions {
+  sttConnOptions: APIConnectOptions;
+  llmConnOptions: APIConnectOptions;
+  ttsConnOptions: APIConnectOptions;
+  maxUnrecoverableErrors: number;
 }
 
-export const DEFAULT_API_CONNECT_OPTIONS = new APIConnectOptions();
+export const DEFAULT_SESSION_CONNECT_OPTIONS: ResolvedSessionConnectOptions = {
+  sttConnOptions: DEFAULT_API_CONNECT_OPTIONS,
+  llmConnOptions: DEFAULT_API_CONNECT_OPTIONS,
+  ttsConnOptions: DEFAULT_API_CONNECT_OPTIONS,
+  maxUnrecoverableErrors: 3,
+};

package/src/voice/agent.ts

@@ -260,27 +260,41 @@ export class Agent<UserData = any> {
       let wrapped_stt = activity.stt;
 
       if (!wrapped_stt.capabilities.streaming) {
-        if (!agent.vad) {
+        const vad = agent.vad || activity.vad;
+        if (!vad) {
           throw new Error(
             'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',
           );
         }
-        wrapped_stt = new STTStreamAdapter(wrapped_stt, agent.vad);
+        wrapped_stt = new STTStreamAdapter(wrapped_stt, vad);
       }
 
-      const stream = wrapped_stt.stream();
+      const connOptions = activity.agentSession.connOptions.sttConnOptions;
+      const stream = wrapped_stt.stream({ connOptions });
       stream.updateInputStream(audio);
 
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.detachInputStream();
+        stream.close();
+      };
+
       return new ReadableStream({
         async start(controller) {
-          for await (const event of stream) {
-            controller.enqueue(event);
+          try {
+            for await (const event of stream) {
+              controller.enqueue(event);
+            }
+            controller.close();
+          } finally {
+            // Always clean up the STT stream, whether it ends naturally or is cancelled
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.detachInputStream();
-          stream.close();
+          cleanup();
         },
       });
     },
@@ -304,22 +318,36 @@ export class Agent<UserData = any> {
 
       // TODO(brian): make parallelToolCalls configurable
       const { toolChoice } = modelSettings;
+      const connOptions = activity.agentSession.connOptions.llmConnOptions;
 
       const stream = activity.llm.chat({
         chatCtx,
         toolCtx,
         toolChoice,
+        connOptions,
         parallelToolCalls: true,
       });
+
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.close();
+      };
+
       return new ReadableStream({
         async start(controller) {
-          for await (const chunk of stream) {
-            controller.enqueue(chunk);
+          try {
+            for await (const chunk of stream) {
+              controller.enqueue(chunk);
+            }
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.close();
+          cleanup();
         },
       });
     },
@@ -340,21 +368,33 @@ export class Agent<UserData = any> {
         wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());
       }
 
-      const stream = wrapped_tts.stream();
+      const connOptions = activity.agentSession.connOptions.ttsConnOptions;
+      const stream = wrapped_tts.stream({ connOptions });
       stream.updateInputStream(text);
 
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.close();
+      };
+
       return new ReadableStream({
         async start(controller) {
-          for await (const chunk of stream) {
-            if (chunk === SynthesizeStream.END_OF_STREAM) {
-              break;
+          try {
+            for await (const chunk of stream) {
+              if (chunk === SynthesizeStream.END_OF_STREAM) {
+                break;
+              }
+              controller.enqueue(chunk.frame);
             }
-            controller.enqueue(chunk.frame);
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.close();
+          cleanup();
         },
       });
     },

package/src/voice/agent_activity.ts

@@ -2259,15 +2259,12 @@ export class AgentActivity implements RecognitionHooks {
       }
       if (this.stt instanceof STT) {
         this.stt.off('metrics_collected', this.onMetricsCollected);
-        await this.stt.close();
       }
       if (this.tts instanceof TTS) {
         this.tts.off('metrics_collected', this.onMetricsCollected);
-        await this.tts.close();
       }
       if (this.vad instanceof VAD) {
         this.vad.off('metrics_collected', this.onMetricsCollected);
-        await this.vad.close();
       }
 
       this.detachAudioInput();