@livekit/agents-plugin-openai 0.7.1 → 0.7.3

package/src/llm.ts

@@ -398,6 +398,7 @@ export class LLM extends llm.LLM {
     temperature = temperature || this.#opts.temperature;
 
     return new LLMStream(
+      this,
       this.#client,
       chatCtx,
       fncCtx,
@@ -416,8 +417,10 @@ export class LLMStream extends llm.LLMStream {
   #client: OpenAI;
   #logger = log();
   #id = randomUUID();
+  label = 'openai.LLMStream';
 
   constructor(
+    llm: LLM,
     client: OpenAI,
     chatCtx: llm.ChatContext,
     fncCtx: llm.FunctionContext | undefined,
@@ -426,7 +429,7 @@ export class LLMStream extends llm.LLMStream {
     temperature?: number,
     n?: number,
   ) {
-    super(chatCtx, fncCtx);
+    super(llm, chatCtx, fncCtx);
     this.#client = client;
     this.#run(opts, n, parallelToolCalls, temperature);
   }

package/src/models.ts

@@ -53,6 +53,7 @@ export type GroqChatModels =
   | 'llama-3.1-405b-reasoning'
   | 'llama-3.1-70b-versatile'
   | 'llama-3.1-8b-instant'
+  | 'llama-3.3-70b-versatile'
   | 'llama3-groq-70b-8192-tool-use-preview'
   | 'llama3-groq-8b-8192-tool-use-preview'
   | 'llama-guard-3-8b'

package/src/realtime/realtime_model.ts

@@ -8,6 +8,7 @@ import {
   llm,
   log,
   mergeFrames,
+  metrics,
   multimodal,
 } from '@livekit/agents';
 import { AudioFrame } from '@livekit/rtc-node';
@@ -40,6 +41,8 @@ export interface RealtimeResponse {
   usage: api_proto.ModelUsage | null;
   output: RealtimeOutput[];
   doneFut: Future;
+  createdTimestamp: number;
+  firstTokenTimestamp?: number;
 }
 
 export interface RealtimeOutput {
@@ -932,6 +935,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       usage: null,
       output: [],
       doneFut: doneFut,
+      createdTimestamp: Date.now(),
     };
     this.#pendingResponses[newResponse.id] = newResponse;
     this.emit('response_created', newResponse);
@@ -946,7 +950,70 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     response.usage = responseData.usage ?? null;
     this.#pendingResponses[responseId] = response;
     response.doneFut.resolve();
+
+    let metricsError: Error | undefined;
+    let cancelled = false;
+    switch (response.status) {
+      case 'failed': {
+        if (response.statusDetails.type !== 'failed') break;
+        const err = response.statusDetails.error;
+        metricsError = new metrics.MultimodalLLMError({
+          type: response.statusDetails.type,
+          code: err?.code,
+          message: err?.message,
+        });
+        this.#logger
+          .child({ code: err?.code, error: err?.message })
+          .error('response generation failed');
+        break;
+      }
+      case 'incomplete': {
+        if (response.statusDetails.type !== 'incomplete') break;
+        const reason = response.statusDetails.reason;
+        metricsError = new metrics.MultimodalLLMError({
+          type: response.statusDetails.type,
+          reason,
+        });
+        this.#logger.child({ reason }).error('response generation incomplete');
+        break;
+      }
+      case 'cancelled': {
+        cancelled = true;
+        break;
+      }
+    }
     this.emit('response_done', response);
+
+    let ttft: number | undefined;
+    if (response.firstTokenTimestamp) {
+      ttft = response.firstTokenTimestamp - response.createdTimestamp;
+    }
+    const duration = Date.now() - response.createdTimestamp;
+
+    const usage = response.usage;
+    const metric: metrics.MultimodalLLMMetrics = {
+      timestamp: response.createdTimestamp,
+      requestId: response.id,
+      ttft: ttft!,
+      duration,
+      cancelled,
+      label: this.constructor.name,
+      completionTokens: usage?.output_tokens || 0,
+      promptTokens: usage?.input_tokens || 0,
+      totalTokens: usage?.total_tokens || 0,
+      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,
+      error: metricsError,
+      inputTokenDetails: {
+        cachedTokens: usage?.input_token_details.cached_tokens || 0,
+        textTokens: usage?.input_token_details.text_tokens || 0,
+        audioTokens: usage?.input_token_details.audio_tokens || 0,
+      },
+      outputTokenDetails: {
+        textTokens: usage?.output_token_details.text_tokens || 0,
+        audioTokens: usage?.output_token_details.audio_tokens || 0,
+      },
+    };
+    this.emit('metrics_collected', metric);
   }
 
   #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {
@@ -1062,6 +1129,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       toolCalls: [],
     };
     output?.content.push(newContent);
+    response!.firstTokenTimestamp = Date.now();
     this.emit('response_content_added', newContent);
   }
 

package/src/stt.ts

@@ -25,6 +25,7 @@ const defaultSTTOptions: STTOptions = {
 export class STT extends stt.STT {
   #opts: STTOptions;
   #client: OpenAI;
+  label = 'openai.STT';
 
   /**
    * Create a new instance of OpenAI STT.
@@ -108,7 +109,7 @@ export class STT extends stt.STT {
     return Buffer.concat([header, Buffer.from(frame.data.buffer)]);
   }
 
-  async recognize(buffer: AudioBuffer, language?: string): Promise<stt.SpeechEvent> {
+  async _recognize(buffer: AudioBuffer, language?: string): Promise<stt.SpeechEvent> {
     const config = this.#sanitizeOptions(language);
     buffer = mergeFrames(buffer);
     const file = new File([this.#createWav(buffer)], 'audio.wav', { type: 'audio/wav' });

package/src/tts.ts

@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { AudioByteStream, tts } from '@livekit/agents';
+import type { AudioFrame } from '@livekit/rtc-node';
+import { randomUUID } from 'crypto';
 import { OpenAI } from 'openai';
 import type { TTSModels, TTSVoices } from './models.js';
 
@@ -27,6 +29,7 @@ const defaultTTSOptions: TTSOptions = {
 export class TTS extends tts.TTS {
   #opts: TTSOptions;
   #client: OpenAI;
+  label = 'openai.TTS';
 
   /**
    * Create a new instance of OpenAI TTS.
@@ -57,6 +60,8 @@ export class TTS extends tts.TTS {
 
   synthesize(text: string): ChunkedStream {
     return new ChunkedStream(
+      this,
+      text,
       this.#client.audio.speech.create({
         input: text,
         model: this.#opts.model,
@@ -73,25 +78,34 @@ export class TTS extends tts.TTS {
 }
 
 export class ChunkedStream extends tts.ChunkedStream {
+  label = 'openai.ChunkedStream';
+
   // set Promise<T> to any because OpenAI returns an annoying Response type
-  constructor(stream: Promise<any>) {
-    super();
+  constructor(tts: TTS, text: string, stream: Promise<any>) {
+    super(text, tts);
     this.#run(stream);
   }
 
   async #run(stream: Promise<Response>) {
     const buffer = await stream.then((r) => r.arrayBuffer());
-    const requestId = crypto.randomUUID();
+    const requestId = randomUUID();
     const audioByteStream = new AudioByteStream(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS);
     const frames = audioByteStream.write(buffer);
 
+    let lastFrame: AudioFrame | undefined;
+    const sendLastFrame = (segmentId: string, final: boolean) => {
+      if (lastFrame) {
+        this.queue.put({ requestId, segmentId, frame: lastFrame, final });
+        lastFrame = undefined;
+      }
+    };
+
     for (const frame of frames) {
-      this.queue.put({
-        frame,
-        requestId,
-        segmentId: requestId,
-      });
+      sendLastFrame(requestId, false);
+      lastFrame = frame;
     }
+    sendLastFrame(requestId, true);
+
     this.queue.close();
   }
 }