@livekit/agents 1.0.38 → 1.0.39

package/src/inference/llm.ts

@@ -17,6 +17,10 @@ import { type AnyString, createAccessToken } from './utils.js';
 const DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';
 
 export type OpenAIModels =
+  | 'openai/gpt-5.2'
+  | 'openai/gpt-5.2-chat-latest'
+  | 'openai/gpt-5.1'
+  | 'openai/gpt-5.1-chat-latest'
   | 'openai/gpt-5'
   | 'openai/gpt-5-mini'
   | 'openai/gpt-5-nano'
@@ -28,19 +32,17 @@ export type OpenAIModels =
   | 'openai/gpt-oss-120b';
 
 export type GoogleModels =
-  | 'google/gemini-3-pro-preview'
-  | 'google/gemini-3-flash-preview'
+  | 'google/gemini-3-pro'
+  | 'google/gemini-3-flash'
   | 'google/gemini-2.5-pro'
   | 'google/gemini-2.5-flash'
   | 'google/gemini-2.5-flash-lite'
   | 'google/gemini-2.0-flash'
   | 'google/gemini-2.0-flash-lite';
 
-export type QwenModels = 'qwen/qwen3-235b-a22b-instruct';
+export type MoonshotModels = 'moonshotai/kimi-k2-instruct';
 
-export type KimiModels = 'moonshotai/kimi-k2-instruct';
-
-export type DeepSeekModels = 'deepseek-ai/deepseek-v3';
+export type DeepSeekModels = 'deepseek-ai/deepseek-v3' | 'deepseek-ai/deepseek-v3.2';
 
 type ChatCompletionPredictionContentParam =
   Expand<OpenAI.Chat.Completions.ChatCompletionPredictionContent>;
@@ -80,13 +82,7 @@ export interface ChatCompletionOptions extends Record<string, unknown> {
   // response_format?: OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format']
 }
 
-export type LLMModels =
-  | OpenAIModels
-  | GoogleModels
-  | QwenModels
-  | KimiModels
-  | DeepSeekModels
-  | AnyString;
+export type LLMModels = OpenAIModels | GoogleModels | MoonshotModels | DeepSeekModels | AnyString;
 
 export interface InferenceLLMOptions {
   model: LLMModels;
@@ -437,7 +433,10 @@ export class LLMStream extends llm.LLMStream {
         if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {
           callChunk = this.createRunningToolCallChunk(id, delta);
           this.toolCallId = this.fncName = this.fncRawArguments = undefined;
-          this.toolExtra = undefined;
+          // Note: We intentionally do NOT reset toolExtra here.
+          // For Gemini 3+, the thought_signature is only provided on the first tool call
+          // in a parallel batch, but must be applied to ALL tool calls in the batch.
+          // We preserve toolExtra so subsequent tool calls inherit the thought_signature.
         }
 
         // Start or continue building the current tool call
@@ -447,9 +446,14 @@ export class LLMStream extends llm.LLMStream {
           this.fncName = tool.function.name;
           this.fncRawArguments = tool.function.arguments || '';
           // Extract extra from tool call (e.g., Google thought signatures)
-          this.toolExtra =
+          // Only update toolExtra if this tool call has extra_content.
+          // Otherwise, inherit from previous tool call (for parallel Gemini tool calls).
+          const newToolExtra =
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             ((tool as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+          if (newToolExtra) {
+            this.toolExtra = newToolExtra;
+          }
         } else if (tool.function.arguments) {
           this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;
         }
@@ -468,6 +472,7 @@ export class LLMStream extends llm.LLMStream {
     ) {
       const callChunk = this.createRunningToolCallChunk(id, delta);
       this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+      // Reset toolExtra at the end of the response (not between parallel tool calls)
       this.toolExtra = undefined;
       return callChunk;
     }

package/src/inference/stt.ts

@@ -25,19 +25,21 @@ import {
 import { type AnyString, connectWs, createAccessToken } from './utils.js';
 
 export type DeepgramModels =
-  | 'deepgram'
+  | 'deepgram/flux-general'
   | 'deepgram/nova-3'
-  | 'deepgram/nova-3-general'
   | 'deepgram/nova-3-medical'
-  | 'deepgram/nova-2-conversationalai'
   | 'deepgram/nova-2'
-  | 'deepgram/nova-2-general'
   | 'deepgram/nova-2-medical'
+  | 'deepgram/nova-2-conversationalai'
   | 'deepgram/nova-2-phonecall';
 
-export type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';
+export type CartesiaModels = 'cartesia/ink-whisper';
+
+export type AssemblyaiModels =
+  | 'assemblyai/universal-streaming'
+  | 'assemblyai/universal-streaming-multilingual';
 
-export type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';
+export type ElevenlabsSTTModels = 'elevenlabs/scribe_v2_realtime';
 
 export interface CartesiaOptions {
   min_volume?: number; // default: not specified
@@ -77,7 +79,7 @@ export type STTLanguages =
   | 'hi'
   | AnyString;
 
-type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels;
+type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels | ElevenlabsSTTModels;
 
 export type STTModels = _STTModels | 'auto' | AnyString;
 

package/src/inference/tts.ts

@@ -23,22 +23,27 @@ import {
 import { type AnyString, connectWs, createAccessToken } from './utils.js';
 
 export type CartesiaModels =
-  | 'cartesia'
-  | 'cartesia/sonic'
+  | 'cartesia/sonic-3'
   | 'cartesia/sonic-2'
-  | 'cartesia/sonic-turbo';
+  | 'cartesia/sonic-turbo'
+  | 'cartesia/sonic';
+
+export type DeepgramTTSModels = 'deepgram/aura' | 'deepgram/aura-2';
 
 export type ElevenlabsModels =
-  | 'elevenlabs'
   | 'elevenlabs/eleven_flash_v2'
   | 'elevenlabs/eleven_flash_v2_5'
   | 'elevenlabs/eleven_turbo_v2'
   | 'elevenlabs/eleven_turbo_v2_5'
   | 'elevenlabs/eleven_multilingual_v2';
 
-export type RimeModels = 'rime' | 'rime/mist' | 'rime/mistv2' | 'rime/arcana';
+export type InworldModels =
+  | 'inworld/inworld-tts-1.5-max'
+  | 'inworld/inworld-tts-1.5-mini'
+  | 'inworld/inworld-tts-1-max'
+  | 'inworld/inworld-tts-1';
 
-export type InworldModels = 'inworld' | 'inworld/inworld-tts-1';
+export type RimeModels = 'rime/arcana' | 'rime/mistv2';
 
 export interface CartesiaOptions {
   duration?: number; // max duration of audio in seconds
@@ -50,25 +55,40 @@ export interface ElevenlabsOptions {
   apply_text_normalization?: 'auto' | 'off' | 'on'; // default: "auto"
 }
 
+export interface DeepgramTTSOptions {}
+
 export interface RimeOptions {}
 
 export interface InworldOptions {}
 
-type _TTSModels = CartesiaModels | ElevenlabsModels | RimeModels | InworldModels;
-
-export type TTSModels = CartesiaModels | ElevenlabsModels | RimeModels | InworldModels | AnyString;
+type _TTSModels =
+  | CartesiaModels
+  | DeepgramTTSModels
+  | ElevenlabsModels
+  | RimeModels
+  | InworldModels;
+
+export type TTSModels =
+  | CartesiaModels
+  | DeepgramTTSModels
+  | ElevenlabsModels
+  | RimeModels
+  | InworldModels
+  | AnyString;
 
 export type ModelWithVoice = `${_TTSModels}:${string}` | TTSModels;
 
 export type TTSOptions<TModel extends TTSModels> = TModel extends CartesiaModels
   ? CartesiaOptions
-  : TModel extends ElevenlabsModels
-    ? ElevenlabsOptions
-    : TModel extends RimeOptions
-      ? RimeOptions
-      : TModel extends InworldOptions
-        ? InworldOptions
-        : Record<string, unknown>;
+  : TModel extends DeepgramTTSModels
+    ? DeepgramTTSOptions
+    : TModel extends ElevenlabsModels
+      ? ElevenlabsOptions
+      : TModel extends RimeModels
+        ? RimeOptions
+        : TModel extends InworldModels
+          ? InworldOptions
+          : Record<string, unknown>;
 
 type TTSEncoding = 'pcm_s16le';
 

package/src/stt/stream_adapter.ts

@@ -4,6 +4,7 @@
 import type { AudioFrame } from '@livekit/rtc-node';
 import { log } from '../log.js';
 import type { APIConnectOptions } from '../types.js';
+import { isStreamClosedError } from '../utils.js';
 import type { VAD, VADStream } from '../vad.js';
 import { VADEventType } from '../vad.js';
 import type { SpeechEvent } from './stt.js';
@@ -68,7 +69,17 @@ export class StreamAdapterWrapper extends SpeechStream {
           this.#vadStream.pushFrame(input);
         }
       }
-      this.#vadStream.endInput();
+
+      // Guard against calling endInput() on already-closed stream
+      // This happens during handover when close() is called while forwardInput is running
+      try {
+        this.#vadStream.endInput();
+      } catch (e) {
+        if (isStreamClosedError(e)) {
+          return;
+        }
+        throw e;
+      }
     };
 
     const recognize = async () => {

package/src/utils.ts

@@ -675,6 +675,20 @@ export class InvalidErrorType extends Error {
   }
 }
 
+/**
+ * Check if an error is a stream closed error that can be safely ignored during cleanup.
+ * This happens during handover/cleanup when close() is called while operations are still running.
+ *
+ * @param error - The error to check.
+ * @returns True if the error is a stream closed error.
+ */
+export function isStreamClosedError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    (error.message === 'Stream is closed' || error.message === 'Input is closed')
+  );
+}
+
 /**
  * In JS an error can be any arbitrary value.
  * This function converts an unknown error to an Error and stores the original value in the error object.

package/src/voice/agent.ts

@@ -325,16 +325,16 @@ export class Agent<UserData = any> {
         );
       }
 
-      // TODO(brian): make parallelToolCalls configurable
       const { toolChoice } = modelSettings;
       const connOptions = activity.agentSession.connOptions.llmConnOptions;
 
+      // parallelToolCalls is not passed here - it will use the value from LLM's modelOptions
+      // This allows users to configure it via: new inference.LLM({ modelOptions: { parallel_tool_calls: false } })
       const stream = activity.llm.chat({
         chatCtx,
         toolCtx,
         toolChoice,
         connOptions,
-        parallelToolCalls: true,
       });
 
       let cleaned = false;

package/src/voice/agent_activity.ts

@@ -194,12 +194,13 @@ export class AgentActivity implements RecognitionHooks {
     if (
       !this.vad &&
       this.stt &&
+      !this.stt.capabilities.streaming &&
       this.llm instanceof LLM &&
       this.allowInterruptions &&
       this.turnDetectionMode === undefined
     ) {
       this.logger.warn(
-        'VAD is not set. Enabling VAD is recommended when using LLM and STT ' +
+        'VAD is not set. Enabling VAD is recommended when using LLM and non-streaming STT ' +
           'for more responsive interruption handling.',
       );
     }
@@ -659,12 +660,14 @@ export class AgentActivity implements RecognitionHooks {
       return;
     }
 
-    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
-      // skip speech handle interruption if server side turn detection is enabled
-      return;
+    if (ev.speechDuration >= this.agentSession.options.minInterruptionDuration) {
+      this.interruptByAudioActivity();
     }
+  }
 
-    if (ev.speechDuration < this.agentSession.options.minInterruptionDuration) {
+  private interruptByAudioActivity(): void {
+    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
+      // skip speech handle interruption if server side turn detection is enabled
       return;
     }
 
@@ -694,7 +697,10 @@ export class AgentActivity implements RecognitionHooks {
       !this._currentSpeech.interrupted &&
       this._currentSpeech.allowInterruptions
     ) {
-      this.logger.info({ 'speech id': this._currentSpeech.id }, 'speech interrupted by VAD');
+      this.logger.info(
+        { 'speech id': this._currentSpeech.id },
+        'speech interrupted by audio activity',
+      );
       this.realtimeSession?.interrupt();
       this._currentSpeech.interrupt();
     }
@@ -715,6 +721,10 @@ export class AgentActivity implements RecognitionHooks {
         // TODO(AJS-106): add multi participant support
       }),
     );
+
+    if (ev.alternatives![0].text) {
+      this.interruptByAudioActivity();
+    }
   }
 
   onFinalTranscript(ev: SpeechEvent): void {
@@ -732,6 +742,20 @@ export class AgentActivity implements RecognitionHooks {
         // TODO(AJS-106): add multi participant support
       }),
     );
+
+    // agent speech might not be interrupted if VAD failed and a final transcript is received
+    // we call interruptByAudioActivity (idempotent) to pause the speech, if possible
+    if (
+      this.audioRecognition &&
+      this.turnDetection !== 'manual' &&
+      this.turnDetection !== 'realtime_llm'
+    ) {
+      this.interruptByAudioActivity();
+
+      // TODO: resume false interruption - schedule a resume timer if interrupted after end_of_speech
+    }
+
+    // TODO: resume false interruption - start interrupt paused speech task
   }
 
   onPreemptiveGeneration(info: PreemptiveGenerationInfo): void {
@@ -1982,7 +2006,6 @@ export class AgentActivity implements RecognitionHooks {
 
     if (audioOutput) {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
-      this.agentSession._updateAgentState('listening');
     }
 
     if (speechHandle.interrupted) {
@@ -2069,17 +2092,15 @@ export class AgentActivity implements RecognitionHooks {
     speechHandle._markGenerationDone();
     // TODO(brian): close tees
 
-    toolOutput.firstToolStartedFuture.await.finally(() => {
-      this.agentSession._updateAgentState('thinking');
-    });
-
     await executeToolsTask.result;
 
+    if (toolOutput.output.length > 0) {
+      this.agentSession._updateAgentState('thinking');
+    } else if (this.agentSession.agentState === 'speaking') {
+      this.agentSession._updateAgentState('listening');
+    }
+
     if (toolOutput.output.length === 0) {
-      // return to listening state for thinking-only turns (no audio output, no tools)
-      if (!speechHandle.interrupted) {
-        this.agentSession._updateAgentState('listening');
-      }
       return;
     }