@livekit/agents-plugin-openai 1.0.5 → 2.0.0

package/src/llm.ts

@@ -2,14 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { APIConnectOptions } from '@livekit/agents';
-import {
-  APIConnectionError,
-  APIStatusError,
-  APITimeoutError,
-  DEFAULT_API_CONNECT_OPTIONS,
-  llm,
-  toError,
-} from '@livekit/agents';
+import { DEFAULT_API_CONNECT_OPTIONS, inference, llm } from '@livekit/agents';
 import { AzureOpenAI, OpenAI } from 'openai';
 import type {
   CerebrasChatModels,
@@ -118,7 +111,7 @@ export class LLM extends llm.LLM {
       temperature?: number;
     } = defaultAzureLLMOptions,
   ): LLM {
-    opts = { ...defaultLLMOptions, ...opts };
+    opts = { ...defaultAzureLLMOptions, ...opts };
     if (opts.apiKey === undefined) {
       throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');
     }
@@ -491,230 +484,17 @@ export class LLM extends llm.LLM {
       extras.tool_choice = toolChoice;
     }
 
-    return new LLMStream(this, {
-      model: this.#opts.model,
+    return new LLMStream(this as unknown as inference.LLM<inference.AzureModels>, {
+      model: this.#opts.model as inference.AzureModels,
       providerFmt: this.#providerFmt,
       client: this.#client,
       chatCtx,
       toolCtx,
       connOptions,
       extraKwargs: extras,
+      gatewayOptions: undefined, // OpenAI plugin doesn't use gateway authentication
     });
   }
 }
 
-export class LLMStream extends llm.LLMStream {
-  #toolCallId?: string;
-  #fncName?: string;
-  #fncRawArguments?: string;
-  #toolIndex?: number;
-  #client: OpenAI;
-  #providerFmt: llm.ProviderFormat;
-  #extraKwargs: Record<string, any>;
-  private model: string | ChatModels;
-
-  constructor(
-    llm: LLM,
-    {
-      model,
-      providerFmt,
-      client,
-      chatCtx,
-      toolCtx,
-      connOptions,
-      extraKwargs,
-    }: {
-      model: string | ChatModels;
-      providerFmt: llm.ProviderFormat;
-      client: OpenAI;
-      chatCtx: llm.ChatContext;
-      toolCtx?: llm.ToolContext;
-      connOptions: APIConnectOptions;
-      extraKwargs: Record<string, any>;
-    },
-  ) {
-    super(llm, { chatCtx, toolCtx, connOptions });
-    this.#client = client;
-    this.#providerFmt = providerFmt;
-    this.#extraKwargs = extraKwargs;
-    this.model = model;
-  }
-
-  protected async run(): Promise<void> {
-    let retryable = true;
-    try {
-      const messages = (await this.chatCtx.toProviderFormat(
-        this.#providerFmt,
-      )) as OpenAI.ChatCompletionMessageParam[];
-
-      const tools = this.toolCtx
-        ? Object.entries(this.toolCtx).map(([name, func]) => ({
-            type: 'function' as const,
-            function: {
-              name,
-              description: func.description,
-              parameters: llm.toJsonSchema(
-                func.parameters,
-              ) as unknown as OpenAI.Chat.Completions.ChatCompletionTool['function']['parameters'],
-            },
-          }))
-        : undefined;
-
-      const stream = await this.#client.chat.completions.create({
-        model: this.model,
-        messages,
-        tools,
-        stream: true,
-        stream_options: { include_usage: true },
-        ...this.#extraKwargs,
-      });
-
-      for await (const chunk of stream) {
-        for (const choice of chunk.choices) {
-          if (this.abortController.signal.aborted) {
-            break;
-          }
-          const chatChunk = this.#parseChoice(chunk.id, choice);
-          if (chatChunk) {
-            retryable = false;
-            this.queue.put(chatChunk);
-          }
-        }
-
-        if (chunk.usage) {
-          const usage = chunk.usage;
-          retryable = false;
-          this.queue.put({
-            id: chunk.id,
-            usage: {
-              completionTokens: usage.completion_tokens,
-              promptTokens: usage.prompt_tokens,
-              promptCachedTokens: usage.prompt_tokens_details?.cached_tokens || 0,
-              totalTokens: usage.total_tokens,
-            },
-          });
-        }
-      }
-    } catch (error) {
-      if (error instanceof OpenAI.APIConnectionTimeoutError) {
-        throw new APITimeoutError({ options: { retryable } });
-      } else if (error instanceof OpenAI.APIError) {
-        throw new APIStatusError({
-          message: error.message,
-          options: {
-            statusCode: error.status,
-            body: error.error,
-            requestId: error.request_id,
-            retryable,
-          },
-        });
-      } else {
-        throw new APIConnectionError({
-          message: toError(error).message,
-          options: { retryable },
-        });
-      }
-    } finally {
-      this.queue.close();
-    }
-  }
-
-  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {
-    const delta = choice.delta;
-
-    // https://github.com/livekit/agents/issues/688
-    // the delta can be None when using Azure OpenAI (content filtering)
-    if (delta === undefined) return undefined;
-
-    if (delta.tool_calls) {
-      // check if we have functions to calls
-      for (const tool of delta.tool_calls) {
-        if (!tool.function) {
-          continue; // oai may add other tools in the future
-        }
-
-        /**
-         * The way OpenAI streams tool calls is a bit tricky.
-         *
-         * For any new tool call, it first emits a delta tool call with id, and function name,
-         * the rest of the delta chunks will only stream the remaining arguments string,
-         * until a new tool call is started or the tool call is finished.
-         * See below for an example.
-         *
-         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)
-         * [ChoiceDeltaToolCall(index=0, id='call_LaVeHWUHpef9K1sd5UO8TtLg', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]
-         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='{"location": "P', name=None), type=None)]
-         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris}', name=None), type=None)]
-         * [ChoiceDeltaToolCall(index=1, id='call_ThU4OmMdQXnnVmpXGOCknXIB', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]
-         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='{"location": "T', name=None), type=None)]
-         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='okyo', name=None), type=None)]
-         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=None), finish_reason='tool_calls', index=0, logprobs=None)
-         */
-        let callChunk: llm.ChatChunk | undefined;
-        // If we have a previous tool call and this is a new one, emit the previous
-        if (this.#toolCallId && tool.id && tool.index !== this.#toolIndex) {
-          callChunk = this.#createRunningToolCallChunk(id, delta);
-          this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;
-        }
-
-        // Start or continue building the current tool call
-        if (tool.function.name) {
-          this.#toolIndex = tool.index;
-          this.#toolCallId = tool.id;
-          this.#fncName = tool.function.name;
-          this.#fncRawArguments = tool.function.arguments || '';
-        } else if (tool.function.arguments) {
-          this.#fncRawArguments = (this.#fncRawArguments || '') + tool.function.arguments;
-        }
-
-        if (callChunk) {
-          return callChunk;
-        }
-      }
-    }
-
-    // If we're done with tool calls, emit the final one
-    if (
-      choice.finish_reason &&
-      ['tool_calls', 'stop'].includes(choice.finish_reason) &&
-      this.#toolCallId !== undefined
-    ) {
-      const callChunk = this.#createRunningToolCallChunk(id, delta);
-      this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;
-      return callChunk;
-    }
-
-    // Regular content message
-    if (!delta.content) {
-      return undefined;
-    }
-
-    return {
-      id,
-      delta: {
-        role: 'assistant',
-        content: delta.content,
-      },
-    };
-  }
-
-  #createRunningToolCallChunk(
-    id: string,
-    delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,
-  ): llm.ChatChunk {
-    return {
-      id,
-      delta: {
-        role: 'assistant',
-        content: delta.content || undefined,
-        toolCalls: [
-          llm.FunctionCall.create({
-            callId: this.#toolCallId!,
-            name: this.#fncName || '',
-            args: this.#fncRawArguments || '',
-          }),
-        ],
-      },
-    };
-  }
-}
+export class LLMStream extends inference.LLMStream<inference.AzureModels> {}

package/src/realtime/realtime_model.ts

@@ -25,6 +25,9 @@ import { AudioFrame, combineAudioFrames } from '@livekit/rtc-node';
 import { type MessageEvent, WebSocket } from 'ws';
 import * as api_proto from './api_proto.js';
 
+// if LK_OPENAI_DEBUG convert it to a number, otherwise set it to 0
+const lkOaiDebug = process.env.LK_OPENAI_DEBUG ? Number(process.env.LK_OPENAI_DEBUG) : 0;
+
 const SAMPLE_RATE = 24000;
 const NUM_CHANNELS = 1;
 const BASE_URL = 'https://api.openai.com/v1';
@@ -640,11 +643,8 @@ export class RealtimeSession extends llm.RealtimeSession {
     } as api_proto.ConversationItemTruncateEvent);
   }
 
-  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs
-  /// with large amounts of base64 audio data.
-  #loggableEvent(
+  private loggableEvent(
     event: api_proto.ClientEvent | api_proto.ServerEvent,
-    maxLength: number = 30,
   ): Record<string, unknown> {
     const untypedEvent: Record<string, unknown> = {};
     for (const [key, value] of Object.entries(event)) {
@@ -654,18 +654,14 @@ export class RealtimeSession extends llm.RealtimeSession {
     }
 
     if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {
-      const truncatedData =
-        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');
-      return { ...untypedEvent, audio: truncatedData };
+      return { ...untypedEvent, audio: '...' };
     }
     if (
       untypedEvent.delta &&
       typeof untypedEvent.delta === 'string' &&
       event.type === 'response.audio.delta'
     ) {
-      const truncatedDelta =
-        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');
-      return { ...untypedEvent, delta: truncatedDelta };
+      return { ...untypedEvent, delta: '...' };
     }
     return untypedEvent;
   }
@@ -699,7 +695,9 @@ export class RealtimeSession extends llm.RealtimeSession {
       azureDeployment: this.oaiRealtimeModel._options.azureDeployment,
     });
 
-    this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);
+    if (lkOaiDebug) {
+      this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);
+    }
 
     return new Promise((resolve, reject) => {
       const ws = new WebSocket(url, { headers });
@@ -849,8 +847,8 @@ export class RealtimeSession extends llm.RealtimeSession {
             break;
           }
 
-          if (event.type !== 'input_audio_buffer.append') {
-            this.#logger.debug(`(client) -> ${JSON.stringify(this.#loggableEvent(event))}`);
+          if (lkOaiDebug) {
+            this.#logger.debug(this.loggableEvent(event), `(client) -> ${event.type}`);
           }
 
           this.emit('openai_client_event_queued', event);
@@ -876,7 +874,9 @@ export class RealtimeSession extends llm.RealtimeSession {
       const event: api_proto.ServerEvent = JSON.parse(message.data as string);
 
       this.emit('openai_server_event_received', event);
-      this.#logger.debug(`(server) <- ${JSON.stringify(this.#loggableEvent(event))}`);
+      if (lkOaiDebug) {
+        this.#logger.debug(this.loggableEvent(event), `(server) <- ${event.type}`);
+      }
 
       switch (event.type) {
         case 'input_audio_buffer.speech_started':
@@ -931,7 +931,9 @@ export class RealtimeSession extends llm.RealtimeSession {
           this.handleError(event);
           break;
         default:
-          this.#logger.debug(`unhandled event: ${event.type}`);
+          if (lkOaiDebug) {
+            this.#logger.debug(`unhandled event: ${event.type}`);
+          }
           break;
       }
     };