@livekit/agents-plugin-openai 1.0.40 → 1.0.42

package/src/realtime/realtime_model_beta.ts

@@ -1,7 +1,6 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import type { metrics } from '@livekit/agents';
 import {
   type APIConnectOptions,
   APIConnectionError,
@@ -11,11 +10,14 @@ import {
   Future,
   Queue,
   Task,
+  type TimedString,
   cancelAndWait,
+  createTimedString,
   delay,
   isAPIError,
   llm,
   log,
+  type metrics,
   shortuuid,
   stream,
 } from '@livekit/agents';
@@ -61,7 +63,7 @@ interface RealtimeOptions {
 
 interface MessageGeneration {
   messageId: string;
-  textChannel: stream.StreamChannel<string>;
+  textChannel: stream.StreamChannel<string | TimedString>;
   audioChannel: stream.StreamChannel<AudioFrame>;
   audioTranscript: string;
   modalities: Future<('text' | 'audio')[]>;
@@ -1107,7 +1109,7 @@ export class RealtimeSession extends llm.RealtimeSession {
     const modalitiesFut = new Future<Modality[]>();
     const itemGeneration: MessageGeneration = {
       messageId: itemId,
-      textChannel: stream.createStreamChannel<string>(),
+      textChannel: stream.createStreamChannel<string | TimedString>(),
       audioChannel: stream.createStreamChannel<AudioFrame>(),
       audioTranscript: '',
       modalities: modalitiesFut,
@@ -1274,16 +1276,19 @@ export class RealtimeSession extends llm.RealtimeSession {
     }
 
     const itemId = event.item_id;
-    const delta = event.delta;
 
-    // TODO (shubhra): add timed string support
+    // When start_time is provided, wrap the delta in a TimedString for aligned transcripts
+    let delta: string | TimedString = event.delta;
+    if (event.start_time !== undefined) {
+      delta = createTimedString({ text: event.delta, startTime: event.start_time });
+    }
 
     const itemGeneration = this.currentGeneration.messages.get(itemId);
     if (!itemGeneration) {
       throw new Error('itemGeneration is not set');
     } else {
       itemGeneration.textChannel.write(delta);
-      itemGeneration.audioTranscript += delta;
+      itemGeneration.audioTranscript += event.delta;
     }
   }
 

package/src/tts.ts

@@ -43,7 +43,7 @@ export class TTS extends tts.TTS {
     super(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS, { streaming: false });
 
     this.#opts = { ...defaultTTSOptions, ...opts };
-    if (this.#opts.apiKey === undefined) {
+    if (this.#opts.apiKey === undefined && !this.#opts.client) {
       throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
     }