@livekit/agents 1.0.24 → 1.0.27

package/src/voice/agent.ts

@@ -260,27 +260,41 @@ export class Agent<UserData = any> {
       let wrapped_stt = activity.stt;
 
       if (!wrapped_stt.capabilities.streaming) {
-        if (!agent.vad) {
+        const vad = agent.vad || activity.vad;
+        if (!vad) {
           throw new Error(
             'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',
           );
         }
-        wrapped_stt = new STTStreamAdapter(wrapped_stt, agent.vad);
+        wrapped_stt = new STTStreamAdapter(wrapped_stt, vad);
       }
 
-      const stream = wrapped_stt.stream();
+      const connOptions = activity.agentSession.connOptions.sttConnOptions;
+      const stream = wrapped_stt.stream({ connOptions });
       stream.updateInputStream(audio);
 
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.detachInputStream();
+        stream.close();
+      };
+
       return new ReadableStream({
         async start(controller) {
-          for await (const event of stream) {
-            controller.enqueue(event);
+          try {
+            for await (const event of stream) {
+              controller.enqueue(event);
+            }
+            controller.close();
+          } finally {
+            // Always clean up the STT stream, whether it ends naturally or is cancelled
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.detachInputStream();
-          stream.close();
+          cleanup();
         },
       });
     },
@@ -304,22 +318,36 @@ export class Agent<UserData = any> {
 
       // TODO(brian): make parallelToolCalls configurable
       const { toolChoice } = modelSettings;
+      const connOptions = activity.agentSession.connOptions.llmConnOptions;
 
       const stream = activity.llm.chat({
         chatCtx,
         toolCtx,
         toolChoice,
+        connOptions,
         parallelToolCalls: true,
       });
+
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.close();
+      };
+
       return new ReadableStream({
         async start(controller) {
-          for await (const chunk of stream) {
-            controller.enqueue(chunk);
+          try {
+            for await (const chunk of stream) {
+              controller.enqueue(chunk);
+            }
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.close();
+          cleanup();
         },
       });
     },
@@ -340,21 +368,33 @@ export class Agent<UserData = any> {
         wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());
       }
 
-      const stream = wrapped_tts.stream();
+      const connOptions = activity.agentSession.connOptions.ttsConnOptions;
+      const stream = wrapped_tts.stream({ connOptions });
       stream.updateInputStream(text);
 
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.close();
+      };
+
       return new ReadableStream({
         async start(controller) {
-          for await (const chunk of stream) {
-            if (chunk === SynthesizeStream.END_OF_STREAM) {
-              break;
+          try {
+            for await (const chunk of stream) {
+              if (chunk === SynthesizeStream.END_OF_STREAM) {
+                break;
+              }
+              controller.enqueue(chunk.frame);
             }
-            controller.enqueue(chunk.frame);
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.close();
+          cleanup();
         },
       });
     },

package/src/voice/agent_activity.ts

@@ -2259,15 +2259,12 @@ export class AgentActivity implements RecognitionHooks {
       }
       if (this.stt instanceof STT) {
         this.stt.off('metrics_collected', this.onMetricsCollected);
-        await this.stt.close();
       }
       if (this.tts instanceof TTS) {
         this.tts.off('metrics_collected', this.onMetricsCollected);
-        await this.tts.close();
       }
       if (this.vad instanceof VAD) {
         this.vad.off('metrics_collected', this.onMetricsCollected);
-        await this.vad.close();
       }
 
       this.detachAudioInput();

package/src/voice/agent_session.ts

@@ -15,7 +15,7 @@ import {
   type STTModelString,
   type TTSModelString,
 } from '../inference/index.js';
-import { getJobContext } from '../job.js';
+import { type JobContext, getJobContext } from '../job.js';
 import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js';
 import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js';
 import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
@@ -25,6 +25,12 @@ import type { STT } from '../stt/index.js';
 import type { STTError } from '../stt/stt.js';
 import { traceTypes, tracer } from '../telemetry/index.js';
 import type { TTS, TTSError } from '../tts/tts.js';
+import {
+  DEFAULT_API_CONNECT_OPTIONS,
+  DEFAULT_SESSION_CONNECT_OPTIONS,
+  type ResolvedSessionConnectOptions,
+  type SessionConnectOptions,
+} from '../types.js';
 import type { VAD } from '../vad.js';
 import type { Agent } from './agent.js';
 import { AgentActivity } from './agent_activity.js';
@@ -40,6 +46,7 @@ import {
   type ErrorEvent,
   type FunctionToolsExecutedEvent,
   type MetricsCollectedEvent,
+  type ShutdownReason,
   type SpeechCreatedEvent,
   type UserInputTranscribedEvent,
   type UserState,
@@ -50,6 +57,7 @@ import {
   createUserStateChangedEvent,
 } from './events.js';
 import { AgentInput, AgentOutput } from './io.js';
+import { RecorderIO } from './recorder_io/index.js';
 import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';
 import type { UnknownUserData } from './run_context.js';
 import type { SpeechHandle } from './speech_handle.js';
@@ -100,6 +108,7 @@ export type AgentSessionOptions<UserData = UnknownUserData> = {
   tts?: TTS | TTSModelString;
   userData?: UserData;
   voiceOptions?: Partial<VoiceOptions>;
+  connOptions?: SessionConnectOptions;
 };
 
 export class AgentSession<
@@ -132,10 +141,20 @@ export class AgentSession<
   private closingTask: Promise<void> | null = null;
   private userAwayTimer: NodeJS.Timeout | null = null;
 
+  // Connection options for STT, LLM, and TTS
+  private _connOptions: ResolvedSessionConnectOptions;
+
+  // Unrecoverable error counts, reset after agent speaking
+  private llmErrorCounts = 0;
+  private ttsErrorCounts = 0;
+
   private sessionSpan?: Span;
   private userSpeakingSpan?: Span;
   private agentSpeakingSpan?: Span;
 
+  /** @internal */
+  _recorderIO?: RecorderIO;
+
   /** @internal */
   rootSpanContext?: Context;
 
@@ -159,8 +178,19 @@ export class AgentSession<
       turnDetection,
       userData,
       voiceOptions = defaultVoiceOptions,
+      connOptions,
     } = opts;
 
+    // Merge user-provided connOptions with defaults
+    this._connOptions = {
+      sttConnOptions: { ...DEFAULT_API_CONNECT_OPTIONS, ...connOptions?.sttConnOptions },
+      llmConnOptions: { ...DEFAULT_API_CONNECT_OPTIONS, ...connOptions?.llmConnOptions },
+      ttsConnOptions: { ...DEFAULT_API_CONNECT_OPTIONS, ...connOptions?.ttsConnOptions },
+      maxUnrecoverableErrors:
+        connOptions?.maxUnrecoverableErrors ??
+        DEFAULT_SESSION_CONNECT_OPTIONS.maxUnrecoverableErrors,
+    };
+
     this.vad = vad;
 
     if (typeof stt === 'string') {
@@ -225,6 +255,11 @@ export class AgentSession<
     return this._chatCtx;
   }
 
+  /** Connection options for STT, LLM, and TTS. */
+  get connOptions(): ResolvedSessionConnectOptions {
+    return this._connOptions;
+  }
+
   set userData(value: UserData) {
     this._userData = value;
   }
@@ -234,14 +269,12 @@ export class AgentSession<
     room,
     inputOptions,
     outputOptions,
-    record,
     span,
   }: {
     agent: Agent;
     room: Room;
     inputOptions?: Partial<RoomInputOptions>;
     outputOptions?: Partial<RoomOutputOptions>;
-    record: boolean;
     span: Span;
   }): Promise<void> {
     span.setAttribute(traceTypes.ATTR_AGENT_LABEL, agent.id);
@@ -275,20 +308,39 @@ export class AgentSession<
     });
     this.roomIO.start();
 
-    const ctx = getJobContext();
-    if (ctx && ctx.room === room && !room.isConnected) {
-      this.logger.debug('Auto-connecting to room via job context');
-      tasks.push(ctx.connect());
+    let ctx: JobContext | undefined = undefined;
+    try {
+      ctx = getJobContext();
+    } catch (error) {
+      // JobContext is not available in evals
+      this.logger.warn('JobContext is not available');
     }
 
-    if (record) {
+    if (ctx) {
+      if (ctx.room === room && !room.isConnected) {
+        this.logger.debug('Auto-connecting to room via job context');
+        tasks.push(ctx.connect());
+      }
+
       if (ctx._primaryAgentSession === undefined) {
         ctx._primaryAgentSession = this;
-      } else {
+      } else if (this._enableRecording) {
         throw new Error(
-          'Only one `AgentSession` can be the primary at a time. If you want to ignore primary designation, use session.start(record=False).',
+          'Only one `AgentSession` can be the primary at a time. If you want to ignore primary designation, use `session.start({ record: false })`.',
         );
       }
+
+      if (this.input.audio && this.output.audio && this._enableRecording) {
+        this._recorderIO = new RecorderIO({ agentSession: this });
+        this.input.audio = this._recorderIO.recordInput(this.input.audio);
+        this.output.audio = this._recorderIO.recordOutput(this.output.audio);
+
+        // Start recording to session directory
+        const sessionDir = ctx.sessionDirectory;
+        if (sessionDir) {
+          tasks.push(this._recorderIO.start(`${sessionDir}/audio.ogg`));
+        }
+      }
     }
 
     // TODO(AJS-265): add shutdown callback to job context
@@ -327,29 +379,29 @@ export class AgentSession<
       return;
     }
 
-    const ctx = getJobContext();
+    let ctx: JobContext | undefined = undefined;
+    try {
+      ctx = getJobContext();
 
-    this.logger.info(
-      { record, enableRecording: ctx.info.job.enableRecording },
-      'Configuring session recording',
-    );
+      if (record === undefined) {
+        record = ctx.job.enableRecording;
+      }
 
-    record = record ?? ctx.info.job.enableRecording;
-    this._enableRecording = record;
+      this._enableRecording = record;
 
-    if (this._enableRecording) {
-      await ctx.initRecording();
+      if (this._enableRecording) {
+        ctx.initRecording();
+      }
+    } catch (error) {
+      // JobContext is not available in evals
+      this.logger.warn('JobContext is not available');
     }
 
-    // Create agent_session as a ROOT span (new trace) to match Python behavior
-    // This creates a separate trace for better cloud dashboard organization
     this.sessionSpan = tracer.startSpan({
       name: 'agent_session',
       context: ROOT_CONTEXT,
     });
 
-    // Set the session span as the active span in the context
-    // This ensures all child spans (agent_turn, user_turn, etc.) are parented to it
     this.rootSpanContext = trace.setSpan(ROOT_CONTEXT, this.sessionSpan);
 
     await this._startImpl({
@@ -357,7 +409,6 @@ export class AgentSession<
       room,
       inputOptions,
       outputOptions,
-      record,
       span: this.sessionSpan,
     });
   }
@@ -397,7 +448,17 @@ export class AgentSession<
       throw new Error('AgentSession is not running');
     }
 
-    return this.activity.say(text, options);
+    const doSay = (activity: AgentActivity) => {
+      return activity.say(text, options);
+    };
+
+    // attach to the session span if called outside of the AgentSession
+    const activeSpan = trace.getActiveSpan();
+    if (!activeSpan && this.rootSpanContext) {
+      return otelContext.with(this.rootSpanContext, () => doSay(this.activity!));
+    }
+
+    return doSay(this.activity);
   }
 
   interrupt() {
@@ -424,14 +485,25 @@ export class AgentSession<
         })
       : undefined;
 
-    if (this.activity.draining) {
-      if (!this.nextActivity) {
-        throw new Error('AgentSession is closing, cannot use generateReply()');
+    const doGenerateReply = (activity: AgentActivity, nextActivity?: AgentActivity) => {
+      if (activity.draining) {
+        if (!nextActivity) {
+          throw new Error('AgentSession is closing, cannot use generateReply()');
+        }
+        return nextActivity.generateReply({ userMessage, ...options });
       }
-      return this.nextActivity.generateReply({ userMessage, ...options });
+      return activity.generateReply({ userMessage, ...options });
+    };
+
+    // attach to the session span if called outside of the AgentSession
+    const activeSpan = trace.getActiveSpan();
+    if (!activeSpan && this.rootSpanContext) {
+      return otelContext.with(this.rootSpanContext, () =>
+        doGenerateReply(this.activity!, this.nextActivity),
+      );
     }
 
-    return this.activity.generateReply({ userMessage, ...options });
+    return doGenerateReply(this.activity!, this.nextActivity);
   }
 
   private async updateActivity(agent: Agent): Promise<void> {
@@ -492,13 +564,22 @@ export class AgentSession<
     await this.closeImpl(CloseReason.USER_INITIATED);
   }
 
+  shutdown(options?: { drain?: boolean; reason?: ShutdownReason }): void {
+    const { drain = true, reason = CloseReason.USER_INITIATED } = options ?? {};
+
+    this._closeSoon({
+      reason,
+      drain,
+    });
+  }
+
   /** @internal */
   _closeSoon({
     reason,
     drain = false,
     error = null,
   }: {
-    reason: CloseReason;
+    reason: ShutdownReason;
     drain?: boolean;
     error?: RealtimeModelError | STTError | TTSError | LLMError | null;
   }): void {
@@ -514,6 +595,19 @@ export class AgentSession<
       return;
     }
 
+    // Track error counts per type to implement max_unrecoverable_errors logic
+    if (error.type === 'llm_error') {
+      this.llmErrorCounts += 1;
+      if (this.llmErrorCounts <= this._connOptions.maxUnrecoverableErrors) {
+        return;
+      }
+    } else if (error.type === 'tts_error') {
+      this.ttsErrorCounts += 1;
+      if (this.ttsErrorCounts <= this._connOptions.maxUnrecoverableErrors) {
+        return;
+      }
+    }
+
     this.logger.error(error, 'AgentSession is closing due to unrecoverable error');
 
     this.closingTask = (async () => {
@@ -541,7 +635,9 @@ export class AgentSession<
     }
 
     if (state === 'speaking') {
-      // TODO(brian): PR4 - Track error counts
+      // Reset error counts when agent starts speaking
+      this.llmErrorCounts = 0;
+      this.ttsErrorCounts = 0;
 
       if (this.agentSpeakingSpan === undefined) {
         this.agentSpeakingSpan = tracer.startSpan({
@@ -657,7 +753,7 @@ export class AgentSession<
   }
 
   private async closeImpl(
-    reason: CloseReason,
+    reason: ShutdownReason,
     error: RealtimeModelError | LLMError | TTSError | STTError | null = null,
     drain: boolean = false,
   ): Promise<void> {
@@ -671,7 +767,7 @@ export class AgentSession<
   }
 
   private async closeImplInner(
-    reason: CloseReason,
+    reason: ShutdownReason,
     error: RealtimeModelError | LLMError | TTSError | STTError | null = null,
     drain: boolean = false,
   ): Promise<void> {
@@ -694,7 +790,16 @@ export class AgentSession<
       await this.activity.drain();
       // wait any uninterruptible speech to finish
       await this.activity.currentSpeech?.waitForPlayout();
-      this.activity.detachAudioInput();
+      try {
+        this.activity.detachAudioInput();
+      } catch (error) {
+        // Ignore detach errors during cleanup - source may not have been set
+      }
+    }
+
+    // Close recorder before detaching inputs/outputs (keep reference for session report)
+    if (this._recorderIO) {
+      await this._recorderIO.close();
     }
 
     // detach the inputs and outputs
@@ -730,6 +835,8 @@ export class AgentSession<
     this.userState = 'listening';
     this._agentState = 'initializing';
     this.rootSpanContext = undefined;
+    this.llmErrorCounts = 0;
+    this.ttsErrorCounts = 0;
 
     this.logger.info({ reason, error }, 'AgentSession closed');
   }

package/src/voice/events.ts

@@ -5,9 +5,10 @@ import type {
   ChatMessage,
   FunctionCall,
   FunctionCallOutput,
+  LLM,
+  RealtimeModel,
   RealtimeModelError,
 } from '../llm/index.js';
-import type { LLM, RealtimeModel } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import type { AgentMetrics } from '../metrics/base.js';
 import type { STT } from '../stt/index.js';
@@ -38,6 +39,8 @@ export enum CloseReason {
   USER_INITIATED = 'user_initiated',
 }
 
+export type ShutdownReason = CloseReason | string;
+
 export type SpeechSource = 'say' | 'generate_reply' | 'tool_response';
 
 export type UserStateChangedEvent = {
@@ -231,12 +234,12 @@ export const createErrorEvent = (
 export type CloseEvent = {
   type: 'close';
   error: RealtimeModelError | STTError | TTSError | LLMError | null;
-  reason: CloseReason;
+  reason: ShutdownReason;
   createdAt: number;
 };
 
 export const createCloseEvent = (
-  reason: CloseReason,
+  reason: ShutdownReason,
   error: RealtimeModelError | STTError | TTSError | LLMError | null = null,
   createdAt: number = Date.now(),
 ): CloseEvent => ({

package/src/voice/generation.ts

@@ -24,7 +24,7 @@ import { isZodSchema, parseZodSchema } from '../llm/zod-utils.js';
 import { log } from '../log.js';
 import { IdentityTransform } from '../stream/identity_transform.js';
 import { traceTypes, tracer } from '../telemetry/index.js';
-import { Future, Task, shortuuid, toError } from '../utils.js';
+import { Future, Task, shortuuid, toError, waitForAbort } from '../utils.js';
 import { type Agent, type ModelSettings, asyncLocalStorage, isStopResponse } from './agent.js';
 import type { AgentSession } from './agent_session.js';
 import type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';
@@ -411,17 +411,19 @@ export function performLLMInference(
         return;
       }
 
+      const abortPromise = waitForAbort(signal);
+
       // TODO(brian): add support for dynamic tools
 
       llmStreamReader = llmStream.getReader();
       while (true) {
-        if (signal.aborted) {
-          break;
-        }
-        const { done, value: chunk } = await llmStreamReader.read();
-        if (done) {
-          break;
-        }
+        if (signal.aborted) break;
+
+        const result = await Promise.race([llmStreamReader.read(), abortPromise]);
+        if (result === undefined) break;
+
+        const { done, value: chunk } = result;
+        if (done) break;
 
         if (typeof chunk === 'string') {
           data.generatedText += chunk;

package/src/voice/io.ts

@@ -28,6 +28,7 @@ export type TTSNode = (
   text: ReadableStream<string>,
   modelSettings: ModelSettings,
 ) => Promise<ReadableStream<AudioFrame> | null>;
+
 export abstract class AudioInput {
   protected deferredStream: DeferredReadableStream<AudioFrame> =
     new DeferredReadableStream<AudioFrame>();
@@ -128,6 +129,24 @@ export abstract class AudioOutput extends EventEmitter {
       this.nextInChain.onDetached();
     }
   }
+
+  /**
+   * Pause the audio playback
+   */
+  pause(): void {
+    if (this.nextInChain) {
+      this.nextInChain.pause();
+    }
+  }
+
+  /**
+   * Resume the audio playback
+   */
+  resume(): void {
+    if (this.nextInChain) {
+      this.nextInChain.resume();
+    }
+  }
 }
 
 export interface PlaybackFinishedEvent {

package/src/voice/recorder_io/index.ts

@@ -0,0 +1,4 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './recorder_io.js';