@livekit/agents-plugin-openai 0.9.3 → 1.0.0-next.0

package/src/models.ts

@@ -3,6 +3,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 export type ChatModels =
+  | 'gpt-5'
+  | 'gpt-5-mini'
+  | 'gpt-5-nano'
+  | 'gpt-4.1'
+  | 'gpt-4.1-mini'
+  | 'gpt-4.1-nano'
   | 'gpt-4o'
   | 'gpt-4o-2024-05-13'
   | 'gpt-4o-mini'
@@ -41,8 +47,26 @@ export type TTSVoices =
   | 'nova'
   | 'onyx'
   | 'sage'
-  | 'shimmer'
-  | 'verse';
+  | 'shimmer';
+
+export type DalleModels = 'dall-e-2' | 'dall-e-3';
+
+export type EmbeddingModels =
+  | 'text-embedding-ada-002'
+  | 'text-embedding-3-small'
+  | 'text-embedding-3-large';
+
+export type AssistantTools = 'code_interpreter' | 'file_search' | 'function';
+
+export type VertexModels =
+  | 'google/gemini-2.0-flash-exp'
+  | 'google/gemini-1.5-flash'
+  | 'google/gemini-1.5-pro'
+  | 'google/gemini-1.0-pro-vision'
+  | 'google/gemini-1.0-pro-vision-001'
+  | 'google/gemini-1.0-pro-002'
+  | 'google/gemini-1.0-pro-001'
+  | 'google/gemini-1.0-pro';
 
 // adapters for OpenAI-compatible LLMs, TTSs, STTs
 
@@ -50,7 +74,16 @@ export type TelnyxChatModels =
   | 'meta-llama/Meta-Llama-3.1-8B-Instruct'
   | 'meta-llama/Meta-Llama-3.1-70B-Instruct';
 
-export type CerebrasChatModels = 'llama3.1-8b' | 'llama3.1-70b';
+export type CerebrasChatModels =
+  | 'llama3.1-8b'
+  | 'llama-3.3-70b'
+  | 'llama-4-scout-17b-16e-instruct'
+  | 'llama-4-maverick-17b-128e-instruct'
+  | 'qwen-3-32b'
+  | 'qwen-3-235b-a22b-instruct-2507'
+  | 'qwen-3-235b-a22b-thinking-2507'
+  | 'qwen-3-coder-480b'
+  | 'gpt-oss-120b';
 
 export type PerplexityChatModels =
   | 'llama-3.1-sonar-small-128k-online'
@@ -62,7 +95,6 @@ export type PerplexityChatModels =
 
 export type GroqChatModels =
   | 'llama-3.1-405b-reasoning'
-  | 'llama-3.1-70b-versatile'
   | 'llama-3.1-8b-instant'
   | 'llama-3.3-70b-versatile'
   | 'llama3-groq-70b-8192-tool-use-preview'
@@ -82,6 +114,40 @@ export type GroqAudioModels =
 export type DeepSeekChatModels = 'deepseek-coder' | 'deepseek-chat';
 
 export type TogetherChatModels =
+  | 'Austism/chronos-hermes-13b'
+  | 'Gryphe/MythoMax-L2-13b'
+  | 'NousResearch/Nous-Capybara-7B-V1p9'
+  | 'NousResearch/Nous-Hermes-2-Mistral-7B-DPO'
+  | 'NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO'
+  | 'NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT'
+  | 'NousResearch/Nous-Hermes-2-Yi-34B'
+  | 'NousResearch/Nous-Hermes-Llama2-13b'
+  | 'NousResearch/Nous-Hermes-llama-2-7b'
+  | 'Open-Orca/Mistral-7B-OpenOrca'
+  | 'Qwen/Qwen1.5-0.5B-Chat'
+  | 'Qwen/Qwen1.5-1.8B-Chat'
+  | 'Qwen/Qwen1.5-110B-Chat'
+  | 'Qwen/Qwen1.5-14B-Chat'
+  | 'Qwen/Qwen1.5-32B-Chat'
+  | 'Qwen/Qwen1.5-4B-Chat'
+  | 'Qwen/Qwen1.5-72B-Chat'
+  | 'Qwen/Qwen1.5-7B-Chat'
+  | 'Qwen/Qwen2-72B-Instruct'
+  | 'Snowflake/snowflake-arctic-instruct'
+  | 'Undi95/ReMM-SLERP-L2-13B'
+  | 'Undi95/Toppy-M-7B'
+  | 'WizardLM/WizardLM-13B-V1.2'
+  | 'allenai/OLMo-7B'
+  | 'allenai/OLMo-7B-Instruct'
+  | 'allenai/OLMo-7B-Twin-2T'
+  | 'codellama/CodeLlama-13b-Instruct-hf'
+  | 'codellama/CodeLlama-34b-Instruct-hf'
+  | 'codellama/CodeLlama-70b-Instruct-hf'
+  | 'codellama/CodeLlama-7b-Instruct-hf'
+  | 'cognitivecomputations/dolphin-2.5-mixtral-8x7b'
+  | 'databricks/dbrx-instruct'
+  | 'deepseek-ai/deepseek-coder-33b-instruct'
+  | 'deepseek-ai/deepseek-llm-67b-chat'
   | 'garage-bAInd/Platypus2-70B-instruct'
   | 'google/gemma-2-27b-it'
   | 'google/gemma-2-9b-it'
@@ -101,6 +167,7 @@ export type TogetherChatModels =
   | 'meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo'
   | 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo'
   | 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo'
+  | 'meta-llama/Llama-3.3-70B-Instruct-Turbo'
   | 'mistralai/Mistral-7B-Instruct-v0.1'
   | 'mistralai/Mistral-7B-Instruct-v0.2'
   | 'mistralai/Mistral-7B-Instruct-v0.3'
@@ -127,10 +194,21 @@ export type OctoChatModels =
   | 'mixtral-8x7b-instruct'
   | 'wizardlm-2-8x22bllamaguard-2-7b';
 
-export type XAIChatModels = 'grok-2' | 'grok-2-mini' | 'grok-2-mini-public' | 'grok-2-public';
+export type XAIChatModels =
+  | 'grok-3'
+  | 'grok-3-fast'
+  | 'grok-3-mini'
+  | 'grok-3-mini-fast'
+  | 'grok-2-vision-1212'
+  | 'grok-2-image-1212'
+  | 'grok-2-1212';
 
 export type MetaChatModels =
   | 'Llama-4-Scout-17B-16E-Instruct-FP8'
   | 'Llama-4-Maverick-17B-128E-Instruct-FP8'
   | 'Llama-3.3-70B-Instruct'
   | 'Llama-3.3-8B-Instruct';
+
+export function supportsReasoningEffort(model: ChatModels | string): boolean {
+  return model === 'gpt-5' || model === 'gpt-5-mini' || model === 'gpt-5-nano';
+}

package/src/realtime/api_proto.ts

@@ -84,7 +84,6 @@ export interface Tool {
     type: 'object';
     properties: {
       [prop: string]: {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         [prop: string]: any;
       };
     };
@@ -97,6 +96,8 @@ export type TurnDetectionType = {
   threshold?: number; // 0.0 to 1.0, default: 0.5
   prefix_padding_ms?: number; // default: 300
   silence_duration_ms?: number; // default: 200
+  create_response?: boolean; // default: true
+  interrupt_response?: boolean; // default: true
 };
 
 export type InputAudioTranscription = {
@@ -228,6 +229,7 @@ export interface ModelUsage {
     cached_tokens_details: {
       text_tokens: number;
       audio_tokens: number;
+      image_tokens: number;
     };
   };
   output_token_details: {
@@ -243,6 +245,7 @@ export interface ResponseResource {
   status_details: ResponseStatusDetails;
   output: ItemResource[];
   usage?: ModelUsage;
+  metadata?: Record<string, string>;
 }
 
 // Client Events
@@ -254,6 +257,7 @@ interface BaseClientEvent {
 export interface SessionUpdateEvent extends BaseClientEvent {
   type: 'session.update';
   session: Partial<{
+    model: Model;
     modalities: ['text', 'audio'] | ['text'];
     instructions: string;
     voice: Voice;
@@ -265,6 +269,7 @@ export interface SessionUpdateEvent extends BaseClientEvent {
     tool_choice: ToolChoice;
     temperature: number;
     max_response_output_tokens?: number | 'inf';
+    speed?: number;
   }>;
 }
 
@@ -282,24 +287,28 @@ export interface InputAudioBufferClearEvent extends BaseClientEvent {
 }
 
 export interface UserItemCreate {
+  id: string;
   type: 'message';
   role: 'user';
   content: (InputTextContent | InputAudioContent)[];
 }
 
 export interface AssistantItemCreate {
+  id: string;
   type: 'message';
   role: 'assistant';
   content: TextContent[];
 }
 
 export interface SystemItemCreate {
+  id: string;
   type: 'message';
   role: 'system';
   content: InputTextContent[];
 }
 
 export interface FunctionCallOutputItemCreate {
+  id: string;
   type: 'function_call_output';
   call_id: string;
   output: string;
@@ -340,6 +349,7 @@ export interface ResponseCreateEvent extends BaseClientEvent {
     tool_choice: ToolChoice;
     temperature: number;
     max_output_tokens: number | 'inf';
+    metadata?: Record<string, string>;
   }>;
 }
 
@@ -412,6 +422,7 @@ export interface InputAudioBufferSpeechStoppedEvent extends BaseServerEvent {
 
 export interface ConversationItemCreatedEvent extends BaseServerEvent {
   type: 'conversation.item.created';
+  previous_item_id: string;
   item: ItemResource;
 }
 
@@ -482,6 +493,7 @@ export interface ResponseContentPartAddedEvent extends BaseServerEvent {
 export interface ResponseContentPartDoneEvent extends BaseServerEvent {
   type: 'response.content_part.done';
   response_id: string;
+  item_id: string;
   output_index: number;
   content_index: number;
   part: ContentPart;
@@ -506,6 +518,7 @@ export interface ResponseTextDoneEvent extends BaseServerEvent {
 export interface ResponseAudioTranscriptDeltaEvent extends BaseServerEvent {
   type: 'response.audio_transcript.delta';
   response_id: string;
+  item_id: string;
   output_index: number;
   content_index: number;
   delta: string;
@@ -522,6 +535,7 @@ export interface ResponseAudioTranscriptDoneEvent extends BaseServerEvent {
 export interface ResponseAudioDeltaEvent extends BaseServerEvent {
   type: 'response.audio.delta';
   response_id: string;
+  item_id: string;
   output_index: number;
   content_index: number;
   delta: AudioBase64Bytes;