@livekit/agents-plugin-openai 0.7.2 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -25,7 +25,7 @@
     "@livekit/agents": "^x",
     "@livekit/agents-plugin-silero": "^x",
     "@livekit/agents-plugins-test": "^x",
-    "@livekit/rtc-node": "^0.12.1",
+    "@livekit/rtc-node": "^0.13.1",
     "@microsoft/api-extractor": "^7.35.0",
     "@types/ws": "^8.5.10",
     "tsup": "^8.3.5",
@@ -37,8 +37,8 @@
     "ws": "^8.16.0"
   },
   "peerDependencies": {
-    "@livekit/rtc-node": "^0.12.1",
-    "@livekit/agents": "^0.5.2x"
+    "@livekit/rtc-node": "^0.13.1",
+    "@livekit/agents": "^0.6.1x"
   },
   "scripts": {
     "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",

package/src/llm.ts

@@ -398,6 +398,7 @@ export class LLM extends llm.LLM {
     temperature = temperature || this.#opts.temperature;
 
     return new LLMStream(
+      this,
       this.#client,
       chatCtx,
       fncCtx,
@@ -416,8 +417,10 @@ export class LLMStream extends llm.LLMStream {
   #client: OpenAI;
   #logger = log();
   #id = randomUUID();
+  label = 'openai.LLMStream';
 
   constructor(
+    llm: LLM,
     client: OpenAI,
     chatCtx: llm.ChatContext,
     fncCtx: llm.FunctionContext | undefined,
@@ -426,7 +429,7 @@ export class LLMStream extends llm.LLMStream {
     temperature?: number,
     n?: number,
   ) {
-    super(chatCtx, fncCtx);
+    super(llm, chatCtx, fncCtx);
     this.#client = client;
     this.#run(opts, n, parallelToolCalls, temperature);
   }

package/src/realtime/realtime_model.ts

@@ -8,6 +8,7 @@ import {
   llm,
   log,
   mergeFrames,
+  metrics,
   multimodal,
 } from '@livekit/agents';
 import { AudioFrame } from '@livekit/rtc-node';
@@ -40,6 +41,8 @@ export interface RealtimeResponse {
   usage: api_proto.ModelUsage | null;
   output: RealtimeOutput[];
   doneFut: Future;
+  createdTimestamp: number;
+  firstTokenTimestamp?: number;
 }
 
 export interface RealtimeOutput {
@@ -62,6 +65,7 @@ export interface RealtimeContent {
   textStream: AsyncIterableQueue<string>;
   audioStream: AsyncIterableQueue<AudioFrame>;
   toolCalls: RealtimeToolCall[];
+  contentType: api_proto.Modality;
 }
 
 export interface RealtimeToolCall {
@@ -666,6 +670,38 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     this.queueMsg(sessionUpdateEvent);
   }
 
+  /** Create an empty audio message with the given duration. */
+  #createEmptyUserAudioMessage(duration: number): llm.ChatMessage {
+    const samples = duration * api_proto.SAMPLE_RATE;
+    return new llm.ChatMessage({
+      role: llm.ChatRole.USER,
+      content: {
+        frame: new AudioFrame(
+          new Int16Array(samples * api_proto.NUM_CHANNELS),
+          api_proto.SAMPLE_RATE,
+          api_proto.NUM_CHANNELS,
+          samples,
+        ),
+      },
+    });
+  }
+
+  /**
+   * Try to recover from a text response to audio mode.
+   *
+   * @remarks
+   * Sometimes the OpenAI Realtime API returns text instead of audio responses.
+   * This method tries to recover from this by requesting a new response after deleting the text
+   * response and creating an empty user audio message.
+   */
+  recoverFromTextResponse(itemId: string) {
+    if (itemId) {
+      this.conversation.item.delete(itemId);
+    }
+    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));
+    this.response.create();
+  }
+
   #start(): Promise<void> {
     return new Promise(async (resolve, reject) => {
       const headers: Record<string, string> = {
@@ -932,6 +968,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       usage: null,
       output: [],
       doneFut: doneFut,
+      createdTimestamp: Date.now(),
     };
     this.#pendingResponses[newResponse.id] = newResponse;
     this.emit('response_created', newResponse);
@@ -946,7 +983,70 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     response.usage = responseData.usage ?? null;
     this.#pendingResponses[responseId] = response;
     response.doneFut.resolve();
+
+    let metricsError: Error | undefined;
+    let cancelled = false;
+    switch (response.status) {
+      case 'failed': {
+        if (response.statusDetails.type !== 'failed') break;
+        const err = response.statusDetails.error;
+        metricsError = new metrics.MultimodalLLMError({
+          type: response.statusDetails.type,
+          code: err?.code,
+          message: err?.message,
+        });
+        this.#logger
+          .child({ code: err?.code, error: err?.message })
+          .error('response generation failed');
+        break;
+      }
+      case 'incomplete': {
+        if (response.statusDetails.type !== 'incomplete') break;
+        const reason = response.statusDetails.reason;
+        metricsError = new metrics.MultimodalLLMError({
+          type: response.statusDetails.type,
+          reason,
+        });
+        this.#logger.child({ reason }).error('response generation incomplete');
+        break;
+      }
+      case 'cancelled': {
+        cancelled = true;
+        break;
+      }
+    }
     this.emit('response_done', response);
+
+    let ttft: number | undefined;
+    if (response.firstTokenTimestamp) {
+      ttft = response.firstTokenTimestamp - response.createdTimestamp;
+    }
+    const duration = Date.now() - response.createdTimestamp;
+
+    const usage = response.usage;
+    const metric: metrics.MultimodalLLMMetrics = {
+      timestamp: response.createdTimestamp,
+      requestId: response.id,
+      ttft: ttft!,
+      duration,
+      cancelled,
+      label: this.constructor.name,
+      completionTokens: usage?.output_tokens || 0,
+      promptTokens: usage?.input_tokens || 0,
+      totalTokens: usage?.total_tokens || 0,
+      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,
+      error: metricsError,
+      inputTokenDetails: {
+        cachedTokens: usage?.input_token_details.cached_tokens || 0,
+        textTokens: usage?.input_token_details.text_tokens || 0,
+        audioTokens: usage?.input_token_details.audio_tokens || 0,
+      },
+      outputTokenDetails: {
+        textTokens: usage?.output_token_details.text_tokens || 0,
+        audioTokens: usage?.output_token_details.audio_tokens || 0,
+      },
+    };
+    this.emit('metrics_collected', metric);
   }
 
   #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {
@@ -1060,8 +1160,10 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       textStream: textStream,
       audioStream: audioStream,
       toolCalls: [],
+      contentType: event.part.type,
     };
     output?.content.push(newContent);
+    response!.firstTokenTimestamp = Date.now();
     this.emit('response_content_added', newContent);
   }
 
@@ -1075,6 +1177,8 @@ export class RealtimeSession extends multimodal.RealtimeSession {
   }
 
   #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {
+    const content = this.#getContent(event);
+    content.text = event.text;
     this.emit('response_text_done', event);
   }
 

package/src/stt.ts

@@ -9,6 +9,7 @@ import type { GroqAudioModels, WhisperModels } from './models.js';
 export interface STTOptions {
   apiKey?: string;
   language: string;
+  prompt?: string;
   detectLanguage: boolean;
   model: WhisperModels | string;
   baseURL?: string;
@@ -25,6 +26,7 @@ const defaultSTTOptions: STTOptions = {
 export class STT extends stt.STT {
   #opts: STTOptions;
   #client: OpenAI;
+  label = 'openai.STT';
 
   /**
    * Create a new instance of OpenAI STT.
@@ -108,7 +110,7 @@ export class STT extends stt.STT {
     return Buffer.concat([header, Buffer.from(frame.data.buffer)]);
   }
 
-  async recognize(buffer: AudioBuffer, language?: string): Promise<stt.SpeechEvent> {
+  async _recognize(buffer: AudioBuffer, language?: string): Promise<stt.SpeechEvent> {
     const config = this.#sanitizeOptions(language);
     buffer = mergeFrames(buffer);
     const file = new File([this.#createWav(buffer)], 'audio.wav', { type: 'audio/wav' });
@@ -116,6 +118,7 @@ export class STT extends stt.STT {
       file,
       model: this.#opts.model,
       language: config.language,
+      prompt: config.prompt,
       response_format: 'json',
     });
 

package/src/tts.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { AudioByteStream, tts } from '@livekit/agents';
+import type { AudioFrame } from '@livekit/rtc-node';
 import { randomUUID } from 'crypto';
 import { OpenAI } from 'openai';
 import type { TTSModels, TTSVoices } from './models.js';
@@ -28,6 +29,7 @@ const defaultTTSOptions: TTSOptions = {
 export class TTS extends tts.TTS {
   #opts: TTSOptions;
   #client: OpenAI;
+  label = 'openai.TTS';
 
   /**
    * Create a new instance of OpenAI TTS.
@@ -58,6 +60,8 @@ export class TTS extends tts.TTS {
 
   synthesize(text: string): ChunkedStream {
     return new ChunkedStream(
+      this,
+      text,
       this.#client.audio.speech.create({
         input: text,
         model: this.#opts.model,
@@ -74,9 +78,11 @@ export class TTS extends tts.TTS {
 }
 
 export class ChunkedStream extends tts.ChunkedStream {
+  label = 'openai.ChunkedStream';
+
   // set Promise<T> to any because OpenAI returns an annoying Response type
-  constructor(stream: Promise<any>) {
-    super();
+  constructor(tts: TTS, text: string, stream: Promise<any>) {
+    super(text, tts);
     this.#run(stream);
   }
 
@@ -86,13 +92,20 @@ export class ChunkedStream extends tts.ChunkedStream {
     const audioByteStream = new AudioByteStream(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS);
     const frames = audioByteStream.write(buffer);
 
+    let lastFrame: AudioFrame | undefined;
+    const sendLastFrame = (segmentId: string, final: boolean) => {
+      if (lastFrame) {
+        this.queue.put({ requestId, segmentId, frame: lastFrame, final });
+        lastFrame = undefined;
+      }
+    };
+
     for (const frame of frames) {
-      this.queue.put({
-        frame,
-        requestId,
-        segmentId: requestId,
-      });
+      sendLastFrame(requestId, false);
+      lastFrame = frame;
     }
+    sendLastFrame(requestId, true);
+
     this.queue.close();
   }
 }