@livekit/agents 1.1.0 → 1.2.1

package/src/inference/llm.ts

@@ -2,100 +2,91 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import OpenAI from 'openai';
-import {
-  APIConnectionError,
-  APIStatusError,
-  APITimeoutError,
-  DEFAULT_API_CONNECT_OPTIONS,
-  toError,
-} from '../index.js';
+import { APIConnectionError, APIStatusError, APITimeoutError } from '../_exceptions.js';
 import * as llm from '../llm/index.js';
 import type { APIConnectOptions } from '../types.js';
-import { type AnyModels, createAccessToken } from './utils.js';
-
-export type AzureModels =
-  // | "azure/gpt-5"
-  // | "azure/gpt-5-mini"
-  // | "azure/gpt-5-nano"
-  'azure/gpt-4.1' | 'azure/gpt-4.1-mini' | 'azure/gpt-4.1-nano';
-// | "azure/gpt-4o"
-// | "azure/gpt-4o-mini"
-
-// https://inference-docs.cerebras.ai/models/overview
-export type CerebrasModels =
-  // production models
-  | 'cerebras/llama3.1-8b'
-  | 'cerebras/llama-3.3-70b'
-  | 'cerebras/llama-4-scout-17b-16e-instruct'
-  | 'cerebras/gpt-oss-120b'
-  | 'cerebras/qwen-3-32b'
-  // preview models
-  | 'cerebras/llama-4-maverick-17b-128e-instruct'
-  | 'cerebras/qwen-3-235b-a22b-instruct-2507';
-
-// https://console.groq.com/docs/models
-export type GroqModels =
-  // production models
-  | 'groq/llama-3.1-8b-instant'
-  | 'groq/llama-3.3-70b-versatile'
-  | 'groq/openai/gpt-oss-120b'
-  | 'groq/openai/gpt-oss-20b'
-  // preview models
-  | 'groq/meta-llama/llama-4-maverick-17b-128e-instruct'
-  | 'groq/meta-llama/llama-4-scout-17b-16e-instruct'
-  | 'groq/qwen/qwen3-32b';
-
-// https://www.baseten.co/library/tag/llms
-export type BasetenModels =
-  | 'baseten/deepseek-ai/DeepSeek-V3-0324'
-  | 'baseten/meta-llama/Llama-4-Scout-17B-16E-Instruct'
-  | 'baseten/meta-llama/Llama-4-Maverick-17B-128E-Instruct'
-  | 'baseten/moonshotai/Kimi-K2-Instruct'
-  | 'baseten/openai/gpt-oss-120b'
-  | 'baseten/Qwen/Qwen3-235B-A22B-Instruct-2507';
-
-export interface AzureOptions {
-  top_p?: number;
-}
-
-export interface CerebrasOptions {
-  top_p?: number;
-}
-
-export interface GroqOptions {
+import { DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { type Expand, toError } from '../utils.js';
+import { type AnyString, createAccessToken, getDefaultInferenceUrl } from './utils.js';
+
+export type OpenAIModels =
+  | 'openai/gpt-5.4'
+  | 'openai/gpt-5.3-chat-latest'
+  | 'openai/gpt-5.2'
+  | 'openai/gpt-5.2-chat-latest'
+  | 'openai/gpt-5.1'
+  | 'openai/gpt-5.1-chat-latest'
+  | 'openai/gpt-5'
+  | 'openai/gpt-5-mini'
+  | 'openai/gpt-5-nano'
+  | 'openai/gpt-4.1'
+  | 'openai/gpt-4.1-mini'
+  | 'openai/gpt-4.1-nano'
+  | 'openai/gpt-4o'
+  | 'openai/gpt-4o-mini'
+  | 'openai/gpt-oss-120b';
+
+export type GoogleModels =
+  | 'google/gemini-3-pro'
+  | 'google/gemini-3-flash'
+  | 'google/gemini-2.5-pro'
+  | 'google/gemini-2.5-flash'
+  | 'google/gemini-2.5-flash-lite'
+  | 'google/gemini-2.0-flash'
+  | 'google/gemini-2.0-flash-lite';
+
+export type MoonshotModels = 'moonshotai/kimi-k2-instruct';
+
+export type DeepSeekModels = 'deepseek-ai/deepseek-v3' | 'deepseek-ai/deepseek-v3.2';
+
+type ChatCompletionPredictionContentParam =
+  Expand<OpenAI.Chat.Completions.ChatCompletionPredictionContent>;
+type WebSearchOptions = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams.WebSearchOptions>;
+type ToolChoice = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams['tool_choice']>;
+type Verbosity = 'low' | 'medium' | 'high';
+
+export interface ChatCompletionOptions extends Record<string, unknown> {
+  frequency_penalty?: number;
+  logit_bias?: Record<string, number>;
+  logprobs?: boolean;
+  max_completion_tokens?: number;
+  max_tokens?: number;
+  metadata?: Record<string, string>;
+  modalities?: Array<'text' | 'audio'>;
+  n?: number;
+  parallel_tool_calls?: boolean;
+  prediction?: ChatCompletionPredictionContentParam | null;
+  presence_penalty?: number;
+  prompt_cache_key?: string;
+  reasoning_effort?: 'minimal' | 'low' | 'medium' | 'high';
+  safety_identifier?: string;
+  seed?: number;
+  service_tier?: 'auto' | 'default' | 'flex' | 'scale' | 'priority';
+  stop?: string | string[];
+  store?: boolean;
+  temperature?: number;
+  top_logprobs?: number;
   top_p?: number;
-}
+  user?: string;
+  verbosity?: Verbosity;
+  web_search_options?: WebSearchOptions;
 
-export interface BasetenOptions {
-  top_p?: number;
+  // livekit-typed arguments
+  tool_choice?: ToolChoice;
+  // TODO(brian): support response format
+  // response_format?: OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format']
 }
 
-export type LLMModels = AzureModels | CerebrasModels | GroqModels | BasetenModels | AnyModels;
+export type LLMModels = OpenAIModels | GoogleModels | MoonshotModels | DeepSeekModels | AnyString;
 
-export type LLMOptions<T extends LLMModels> = T extends AzureModels
-  ? AzureOptions
-  : T extends CerebrasModels
-    ? CerebrasOptions
-    : T extends GroqOptions
-      ? GroqOptions
-      : T extends BasetenOptions
-        ? BasetenOptions
-        : Record<string, unknown>;
-
-export type Verbosity = 'low' | 'medium' | 'high';
-const DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';
-
-export interface InferenceLLMOptions<TModel extends LLMModels> {
-  model: TModel;
-  temperature?: number;
-  parallelToolCalls?: boolean;
-  toolChoice?: llm.ToolChoice;
-  maxCompletionTokens?: number;
+export interface InferenceLLMOptions {
+  model: LLMModels;
+  provider?: string;
   baseURL: string;
   apiKey: string;
   apiSecret: string;
-  verbosity?: Verbosity;
-  extraKwargs: Record<string, any>; // eslint-disable-line @typescript-eslint/no-explicit-any
+  modelOptions: ChatCompletionOptions;
+  strictToolSchema?: boolean;
 }
 
 export interface GatewayOptions {
@@ -103,42 +94,35 @@ export interface GatewayOptions {
   apiSecret: string;
 }
 
-export class LLM<TModel extends LLMModels> extends llm.LLM {
+/**
+ * Livekit Cloud Inference LLM
+ */
+export class LLM extends llm.LLM {
   private client: OpenAI;
-  private opts: InferenceLLMOptions<TModel>;
+  private opts: InferenceLLMOptions;
 
   constructor(opts: {
-    model: TModel;
-    temperature?: number;
-    parallelToolCalls?: boolean;
-    toolChoice?: llm.ToolChoice;
-    maxCompletionTokens?: number;
+    model: LLMModels;
+    provider?: string;
     baseURL?: string;
     apiKey?: string;
     apiSecret?: string;
-    maxRetries?: number;
-    timeout?: number;
-    verbosity?: Verbosity;
-    extraKwargs?: LLMOptions<TModel>;
+    modelOptions?: InferenceLLMOptions['modelOptions'];
+    strictToolSchema?: boolean;
   }) {
     super();
 
     const {
       model,
-      temperature,
-      parallelToolCalls,
-      toolChoice,
-      maxCompletionTokens,
+      provider,
       baseURL,
       apiKey,
       apiSecret,
-      maxRetries,
-      timeout,
-      verbosity,
-      extraKwargs,
+      modelOptions,
+      strictToolSchema = false,
     } = opts;
 
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || getDefaultInferenceUrl();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');
@@ -152,21 +136,17 @@ export class LLM<TModel extends LLMModels> extends llm.LLM {
 
     this.opts = {
       model,
-      temperature,
-      parallelToolCalls,
-      toolChoice,
-      verbosity,
-      maxCompletionTokens,
+      provider,
       baseURL: lkBaseURL,
       apiKey: lkApiKey,
       apiSecret: lkApiSecret,
-      extraKwargs: extraKwargs || {},
+      modelOptions: modelOptions || {},
+      strictToolSchema,
     };
 
     this.client = new OpenAI({
       baseURL: this.opts.baseURL,
-      maxRetries: maxRetries || 0,
-      timeout: timeout || 15000,
+      apiKey: '', // leave a temporary empty string to avoid OpenAI complain about missing key
     });
   }
 
@@ -178,6 +158,14 @@ export class LLM<TModel extends LLMModels> extends llm.LLM {
     return this.opts.model;
   }
 
+  get provider(): string {
+    return 'livekit';
+  }
+
+  static fromModelString(modelString: string): LLM {
+    return new LLM({ model: modelString });
+  }
+
   chat({
     chatCtx,
     toolCtx,
@@ -194,42 +182,40 @@ export class LLM<TModel extends LLMModels> extends llm.LLM {
     toolChoice?: llm.ToolChoice;
     // TODO(AJS-270): Add responseFormat parameter
     extraKwargs?: Record<string, unknown>;
-  }): LLMStream<TModel> {
-    let extras: Record<string, unknown> = { ...(extraKwargs || {}) };
-
-    if (this.opts.maxCompletionTokens !== undefined) {
-      extras.max_completion_tokens = this.opts.maxCompletionTokens;
-    }
-    if (this.opts.temperature !== undefined) {
-      extras.temperature = this.opts.temperature;
-    }
-    if (this.opts.verbosity !== undefined) {
-      extras.verbosity = this.opts.verbosity;
-    }
+  }): LLMStream {
+    let modelOptions: Record<string, unknown> = { ...(extraKwargs || {}) };
 
     parallelToolCalls =
-      parallelToolCalls !== undefined ? parallelToolCalls : this.opts.parallelToolCalls;
+      parallelToolCalls !== undefined
+        ? parallelToolCalls
+        : this.opts.modelOptions.parallel_tool_calls;
+
     if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) {
-      extras.parallel_tool_calls = parallelToolCalls;
+      modelOptions.parallel_tool_calls = parallelToolCalls;
     }
 
-    toolChoice = toolChoice !== undefined ? toolChoice : this.opts.toolChoice;
+    toolChoice =
+      toolChoice !== undefined
+        ? toolChoice
+        : (this.opts.modelOptions.tool_choice as llm.ToolChoice | undefined);
+
     if (toolChoice) {
-      extras.tool_choice = toolChoice;
+      modelOptions.tool_choice = toolChoice as ToolChoice;
     }
 
     // TODO(AJS-270): Add response_format support here
 
-    extras = { ...extras, ...this.opts.extraKwargs };
+    modelOptions = { ...modelOptions, ...this.opts.modelOptions };
 
     return new LLMStream(this, {
       model: this.opts.model,
-      providerFmt: 'openai',
+      provider: this.opts.provider,
       client: this.client,
       chatCtx,
       toolCtx,
       connOptions,
-      extraKwargs: extras,
+      modelOptions,
+      strictToolSchema: this.opts.strictToolSchema ?? false, // default to false if not set
       gatewayOptions: {
         apiKey: this.opts.apiKey,
         apiSecret: this.opts.apiSecret,
@@ -238,46 +224,55 @@ export class LLM<TModel extends LLMModels> extends llm.LLM {
   }
 }
 
-export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
-  private model: TModel;
+export class LLMStream extends llm.LLMStream {
+  private model: LLMModels;
+  private provider?: string;
   private providerFmt: llm.ProviderFormat;
   private client: OpenAI;
-  private extraKwargs: Record<string, unknown>;
+  private modelOptions: Record<string, unknown>;
+  private strictToolSchema: boolean;
 
   private gatewayOptions?: GatewayOptions;
   private toolCallId?: string;
   private toolIndex?: number;
   private fncName?: string;
   private fncRawArguments?: string;
+  private toolExtra?: Record<string, unknown>;
 
   constructor(
-    llm: LLM<TModel>,
+    llm: LLM,
     {
       model,
-      providerFmt,
+      provider,
       client,
       chatCtx,
       toolCtx,
       gatewayOptions,
       connOptions,
-      extraKwargs,
+      modelOptions,
+      providerFmt,
+      strictToolSchema,
     }: {
-      model: TModel;
-      providerFmt: llm.ProviderFormat;
+      model: LLMModels;
+      provider?: string;
       client: OpenAI;
       chatCtx: llm.ChatContext;
       toolCtx?: llm.ToolContext;
       gatewayOptions?: GatewayOptions;
       connOptions: APIConnectOptions;
-      extraKwargs: Record<string, any>;
+      modelOptions: Record<string, unknown>;
+      providerFmt?: llm.ProviderFormat;
+      strictToolSchema: boolean;
     },
   ) {
     super(llm, { chatCtx, toolCtx, connOptions });
     this.client = client;
     this.gatewayOptions = gatewayOptions;
-    this.providerFmt = providerFmt;
-    this.extraKwargs = extraKwargs;
+    this.provider = provider;
+    this.providerFmt = providerFmt || 'openai';
+    this.modelOptions = modelOptions;
     this.model = model;
+    this.strictToolSchema = strictToolSchema;
   }
 
   protected async run(): Promise<void> {
@@ -285,6 +280,7 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
     // (defined inside the run method to make sure the state is reset for each run/attempt)
     let retryable = true;
     this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = undefined;
+    this.toolExtra = undefined;
 
     try {
       const messages = (await this.chatCtx.toProviderFormat(
@@ -292,21 +288,31 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
       )) as OpenAI.ChatCompletionMessageParam[];
 
       const tools = this.toolCtx
-        ? Object.entries(this.toolCtx).map(([name, func]) => ({
-            type: 'function' as const,
-            function: {
-              name,
-              description: func.description,
-              parameters: llm.toJsonSchema(
-                func.parameters,
-              ) as unknown as OpenAI.Chat.Completions.ChatCompletionTool['function']['parameters'],
-            },
-          }))
+        ? Object.entries(this.toolCtx).map(([name, func]) => {
+            const oaiParams = {
+              type: 'function' as const,
+              function: {
+                name,
+                description: func.description,
+                parameters: llm.toJsonSchema(
+                  func.parameters,
+                  true,
+                  this.strictToolSchema,
+                ) as unknown as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function']['parameters'],
+              } as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function'],
+            };
+
+            if (this.strictToolSchema) {
+              oaiParams.function.strict = true;
+            }
+
+            return oaiParams;
+          })
         : undefined;
 
-      const requestExtras: Record<string, unknown> = { ...this.extraKwargs };
+      const requestOptions: Record<string, unknown> = { ...this.modelOptions };
       if (!tools) {
-        delete requestExtras.tool_choice;
+        delete requestOptions.tool_choice;
       }
 
       // Dynamically set the access token for the LiveKit Agent Gateway API
@@ -317,6 +323,14 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
         );
       }
 
+      if (this.provider) {
+        const extraHeaders = requestOptions.extra_headers
+          ? (requestOptions.extra_headers as Record<string, string>)
+          : {};
+        extraHeaders['X-LiveKit-Inference-Provider'] = this.provider;
+        requestOptions.extra_headers = extraHeaders;
+      }
+
       const stream = await this.client.chat.completions.create(
         {
           model: this.model,
@@ -324,7 +338,7 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
           tools,
           stream: true,
           stream_options: { include_usage: true },
-          ...requestExtras,
+          ...requestOptions,
         },
         {
           timeout: this.connOptions.timeoutMs,
@@ -366,7 +380,7 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
           options: {
             statusCode: error.status,
             body: error.error,
-            requestId: error.request_id,
+            requestId: error.requestID,
             retryable,
           },
         });
@@ -376,8 +390,6 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
           options: { retryable },
         });
       }
-    } finally {
-      this.queue.close();
     }
   }
 
@@ -408,10 +420,10 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
          *
          * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)
          * [ChoiceDeltaToolCall(index=0, id='call_LaVeHWUHpef9K1sd5UO8TtLg', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]
-         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='{"location": "P', name=None), type=None)]
-         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris}', name=None), type=None)]
+         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='\{"location": "P', name=None), type=None)]
+         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris\}', name=None), type=None)]
          * [ChoiceDeltaToolCall(index=1, id='call_ThU4OmMdQXnnVmpXGOCknXIB', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]
-         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='{"location": "T', name=None), type=None)]
+         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='\{"location": "T', name=None), type=None)]
          * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='okyo', name=None), type=None)]
          * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=None), finish_reason='tool_calls', index=0, logprobs=None)
          */
@@ -420,6 +432,10 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
         if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {
           callChunk = this.createRunningToolCallChunk(id, delta);
           this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+          // Note: We intentionally do NOT reset toolExtra here.
+          // For Gemini 3+, the thought_signature is only provided on the first tool call
+          // in a parallel batch, but must be applied to ALL tool calls in the batch.
+          // We preserve toolExtra so subsequent tool calls inherit the thought_signature.
         }
 
         // Start or continue building the current tool call
@@ -428,6 +444,15 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
           this.toolCallId = tool.id;
           this.fncName = tool.function.name;
           this.fncRawArguments = tool.function.arguments || '';
+          // Extract extra from tool call (e.g., Google thought signatures)
+          // Only update toolExtra if this tool call has extra_content.
+          // Otherwise, inherit from previous tool call (for parallel Gemini tool calls).
+          const newToolExtra =
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ((tool as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+          if (newToolExtra) {
+            this.toolExtra = newToolExtra;
+          }
         } else if (tool.function.arguments) {
           this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;
         }
@@ -446,11 +471,18 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
     ) {
       const callChunk = this.createRunningToolCallChunk(id, delta);
       this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+      // Reset toolExtra at the end of the response (not between parallel tool calls)
+      this.toolExtra = undefined;
       return callChunk;
     }
 
+    // Extract extra from delta (e.g., Google thought signatures on text parts)
+    const deltaExtra =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+
     // Regular content message
-    if (!delta.content) {
+    if (!delta.content && !deltaExtra) {
       return undefined;
     }
 
@@ -458,7 +490,8 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
       id,
       delta: {
         role: 'assistant',
-        content: delta.content,
+        content: delta.content || undefined,
+        extra: deltaExtra,
       },
     };
   }
@@ -467,19 +500,37 @@ export class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
     id: string,
     delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,
   ): llm.ChatChunk {
+    const toolExtra = this.toolExtra ? { ...this.toolExtra } : {};
+    const thoughtSignature = this.extractThoughtSignature(toolExtra);
+    const deltaExtra =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+
     return {
       id,
       delta: {
         role: 'assistant',
         content: delta.content || undefined,
+        extra: deltaExtra,
         toolCalls: [
           llm.FunctionCall.create({
             callId: this.toolCallId || '',
             name: this.fncName || '',
             args: this.fncRawArguments || '',
+            extra: toolExtra,
+            thoughtSignature,
           }),
         ],
       },
     };
   }
+
+  private extractThoughtSignature(extra?: Record<string, unknown>): string | undefined {
+    const googleExtra = extra?.google;
+    if (googleExtra && typeof googleExtra === 'object') {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return (googleExtra as any).thoughtSignature || (googleExtra as any).thought_signature;
+    }
+    return undefined;
+  }
 }