@livekit/agents-plugin-openai 1.0.50 → 1.1.0

package/src/responses/llm.ts

@@ -8,12 +8,14 @@ import {
   APITimeoutError,
   DEFAULT_API_CONNECT_OPTIONS,
   llm,
+  log,
   toError,
 } from '@livekit/agents';
 import OpenAI from 'openai';
 import type { ChatModels } from '../models.js';
+import { WSLLM } from '../ws/llm.js';
 
-interface LLMOptions {
+export interface LLMOptions {
   model: string | ChatModels;
   apiKey?: string;
   baseURL?: string;
@@ -24,30 +26,32 @@ interface LLMOptions {
   store?: boolean;
   metadata?: Record<string, string>;
   strictToolSchema?: boolean;
+
+  /**
+   * Whether to use the WebSocket API.
+   * @default true
+   */
+  useWebSocket?: boolean;
 }
 
+type HttpLLMOptions = Omit<LLMOptions, 'useWebSocket'>;
+
 const defaultLLMOptions: LLMOptions = {
   model: 'gpt-4.1',
   apiKey: process.env.OPENAI_API_KEY,
   strictToolSchema: true,
+  useWebSocket: true,
 };
 
-export class LLM extends llm.LLM {
+class ResponsesHttpLLM extends llm.LLM {
   #client: OpenAI;
-  #opts: LLMOptions;
+  #opts: HttpLLMOptions;
 
-  /**
-   * Create a new instance of OpenAI Responses LLM.
-   *
-   * @remarks
-   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
-   * `OPENAI_API_KEY` environment variable.
-   */
-  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {
+  constructor(opts: Partial<HttpLLMOptions> = defaultLLMOptions) {
     super();
 
     this.#opts = { ...defaultLLMOptions, ...opts };
-    if (this.#opts.apiKey === undefined) {
+    if (this.#opts.apiKey === undefined && this.#opts.client === undefined) {
       throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
     }
 
@@ -59,15 +63,15 @@ export class LLM extends llm.LLM {
       });
   }
 
-  label(): string {
+  override label(): string {
     return 'openai.responses.LLM';
   }
 
-  get model(): string {
+  override get model(): string {
     return this.#opts.model;
   }
 
-  chat({
+  override chat({
     chatCtx,
     toolCtx,
     connOptions = DEFAULT_API_CONNECT_OPTIONS,
@@ -81,7 +85,7 @@ export class LLM extends llm.LLM {
     parallelToolCalls?: boolean;
     toolChoice?: llm.ToolChoice;
     extraKwargs?: Record<string, unknown>;
-  }): LLMStream {
+  }): ResponsesHttpLLMStream {
     const modelOptions: Record<string, unknown> = { ...(extraKwargs || {}) };
 
     parallelToolCalls =
@@ -110,7 +114,7 @@ export class LLM extends llm.LLM {
       modelOptions.metadata = this.#opts.metadata;
     }
 
-    return new LLMStream(this, {
+    return new ResponsesHttpLLMStream(this, {
       model: this.#opts.model,
       client: this.#client,
       chatCtx,
@@ -122,7 +126,7 @@ export class LLM extends llm.LLM {
   }
 }
 
-export class LLMStream extends llm.LLMStream {
+class ResponsesHttpLLMStream extends llm.LLMStream {
   private model: string | ChatModels;
   private client: OpenAI;
   private modelOptions: Record<string, unknown>;
@@ -130,7 +134,7 @@ export class LLMStream extends llm.LLMStream {
   private responseId: string;
 
   constructor(
-    llm: LLM,
+    llm: ResponsesHttpLLM,
     {
       model,
       client,
@@ -325,3 +329,85 @@ export class LLMStream extends llm.LLMStream {
     return undefined;
   }
 }
+
+export class LLM extends llm.LLM {
+  #opts: LLMOptions;
+  #llm: llm.LLM;
+  #logger = log();
+
+  /**
+   * Create a new instance of OpenAI Responses LLM.
+   *
+   * @remarks
+   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+   * `OPENAI_API_KEY` environment variable.
+   */
+  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {
+    super();
+
+    this.#opts = { ...defaultLLMOptions, ...opts };
+    const { useWebSocket, client, ...baseOpts } = this.#opts;
+
+    if (useWebSocket) {
+      if (client !== undefined) {
+        this.#logger.warn(
+          'WebSocket mode does not support custom client; provided client will be ignored',
+        );
+      }
+      this.#llm = new WSLLM(baseOpts);
+    } else {
+      this.#llm = new ResponsesHttpLLM({ ...baseOpts, client });
+    }
+
+    // Forward events from the inner delegate so consumers listening on this
+    // wrapper instance (e.g. AgentActivity) receive them.
+    this.#llm.on('metrics_collected', (metrics) => this.emit('metrics_collected', metrics));
+    this.#llm.on('error', (error) => this.emit('error', error));
+  }
+
+  override label(): string {
+    return this.#llm.label();
+  }
+
+  override get model(): string {
+    return this.#llm.model;
+  }
+
+  override prewarm(): void {
+    this.#llm.prewarm();
+  }
+
+  // Ref: python livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/responses/llm.py - 229-233 lines
+  override async aclose(): Promise<void> {
+    await this.#llm.aclose();
+  }
+
+  async close(): Promise<void> {
+    await this.aclose();
+  }
+
+  override chat({
+    chatCtx,
+    toolCtx,
+    connOptions = DEFAULT_API_CONNECT_OPTIONS,
+    parallelToolCalls,
+    toolChoice,
+    extraKwargs,
+  }: {
+    chatCtx: llm.ChatContext;
+    toolCtx?: llm.ToolContext;
+    connOptions?: APIConnectOptions;
+    parallelToolCalls?: boolean;
+    toolChoice?: llm.ToolChoice;
+    extraKwargs?: Record<string, unknown>;
+  }): llm.LLMStream {
+    return this.#llm.chat({
+      chatCtx,
+      toolCtx,
+      connOptions,
+      parallelToolCalls,
+      toolChoice,
+      extraKwargs,
+    });
+  }
+}

package/src/stt.test.ts

@@ -3,9 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0
 import { VAD } from '@livekit/agents-plugin-silero';
 import { stt } from '@livekit/agents-plugins-test';
-import { describe } from 'vitest';
+import { describe, it } from 'vitest';
 import { STT } from './stt.js';
 
-describe('OpenAI', async () => {
-  await stt(new STT(), await VAD.load(), { streaming: false });
-});
+const hasOpenAIApiKey = Boolean(process.env.OPENAI_API_KEY);
+
+if (hasOpenAIApiKey) {
+  describe('OpenAI', async () => {
+    await stt(new STT(), await VAD.load(), { streaming: false });
+  });
+} else {
+  describe('OpenAI', () => {
+    it.skip('requires OPENAI_API_KEY', () => {});
+  });
+}

package/src/stt.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { type AudioBuffer, mergeFrames, stt } from '@livekit/agents';
+import { type AudioBuffer, mergeFrames, normalizeLanguage, stt } from '@livekit/agents';
 import type { AudioFrame } from '@livekit/rtc-node';
 import { OpenAI } from 'openai';
 import type { GroqAudioModels, WhisperModels } from './models.js';
@@ -28,6 +28,19 @@ export class STT extends stt.STT {
   #client: OpenAI;
   label = 'openai.STT';
 
+  get model(): string {
+    return this.#opts.model;
+  }
+
+  get provider(): string {
+    try {
+      const url = new URL(this.#client.baseURL);
+      return url.host;
+    } catch {
+      return 'api.openai.com';
+    }
+  }
+
   /**
    * Create a new instance of OpenAI STT.
    *
@@ -38,7 +51,11 @@ export class STT extends stt.STT {
   constructor(opts: Partial<STTOptions> = defaultSTTOptions) {
     super({ streaming: false, interimResults: false, alignedTranscript: false });
 
-    this.#opts = { ...defaultSTTOptions, ...opts };
+    this.#opts = {
+      ...defaultSTTOptions,
+      ...opts,
+      language: normalizeLanguage(opts.language ?? defaultSTTOptions.language),
+    };
     if (this.#opts.apiKey === undefined) {
       throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
     }
@@ -113,7 +130,7 @@ export class STT extends stt.STT {
 
   #sanitizeOptions(language?: string): STTOptions {
     if (language) {
-      return { ...this.#opts, language };
+      return { ...this.#opts, language: normalizeLanguage(language) };
     } else {
       return this.#opts;
     }
@@ -165,7 +182,7 @@ export class STT extends stt.STT {
       alternatives: [
         {
           text: resp.text || '',
-          language: config.language || '',
+          language: normalizeLanguage(config.language || ''),
           startTime: 0,
           endTime: 0,
           confidence: 0,

package/src/tts.test.ts

@@ -2,10 +2,18 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { tts } from '@livekit/agents-plugins-test';
-import { describe } from 'vitest';
+import { describe, it } from 'vitest';
 import { STT } from './stt.js';
 import { TTS } from './tts.js';
 
-describe('OpenAI', async () => {
-  await tts(new TTS(), new STT(), { streaming: false });
-});
+const hasOpenAIApiKey = Boolean(process.env.OPENAI_API_KEY);
+
+if (hasOpenAIApiKey) {
+  describe('OpenAI', async () => {
+    await tts(new TTS(), new STT(), { streaming: false });
+  });
+} else {
+  describe('OpenAI', () => {
+    it.skip('requires OPENAI_API_KEY', () => {});
+  });
+}

package/src/tts.ts

@@ -32,6 +32,19 @@ export class TTS extends tts.TTS {
   label = 'openai.TTS';
   private abortController = new AbortController();
 
+  get model(): string {
+    return this.#opts.model;
+  }
+
+  get provider(): string {
+    try {
+      const url = new URL(this.#client.baseURL);
+      return url.host;
+    } catch {
+      return 'api.openai.com';
+    }
+  }
+
   /**
    * Create a new instance of OpenAI TTS.
    *

package/src/ws/index.ts

@@ -0,0 +1,17 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export { ResponsesWebSocket } from './llm.js';
+export type {
+  WsErrorEvent,
+  WsFunctionCallItem,
+  WsOutputItem,
+  WsOutputItemDoneEvent,
+  WsOutputTextDeltaEvent,
+  WsResponseCompletedEvent,
+  WsResponseCreateEvent,
+  WsResponseCreatedEvent,
+  WsResponseFailedEvent,
+  WsServerEvent,
+} from './types.js';

package/src/ws/llm.test.ts

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { initializeLogger } from '@livekit/agents';
+import { llm, llmStrict } from '@livekit/agents-plugins-test';
+import { describe } from 'vitest';
+import { LLM } from '../responses/llm.js';
+
+initializeLogger({ level: 'silent', pretty: false });
+
+describe('OpenAI Responses WS wrapper', async () => {
+  await llm(
+    new LLM({
+      temperature: 0,
+      strictToolSchema: false,
+      useWebSocket: true,
+    }),
+    true,
+  );
+});
+
+describe('OpenAI Responses WS wrapper strict tool schema', async () => {
+  await llmStrict(
+    new LLM({
+      temperature: 0,
+      strictToolSchema: true,
+      useWebSocket: true,
+    }),
+  );
+});