@livekit/agents 0.6.0 → 0.6.2

package/src/multimodal/multimodal_agent.ts

@@ -36,6 +36,7 @@ export abstract class RealtimeSession extends EventEmitter {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer
   abstract fncCtx: llm.FunctionContext | undefined;
+  abstract recoverFromTextResponse(itemId: string): void;
 }
 
 /**
@@ -63,19 +64,25 @@ export class MultimodalAgent extends EventEmitter {
   subscribedTrack: RemoteAudioTrack | null = null;
   readMicroTask: Promise<void> | null = null;
 
+  #textResponseRetries = 0;
+  #maxTextResponseRetries: number;
+
   constructor({
     model,
     chatCtx,
     fncCtx,
+    maxTextResponseRetries = 5,
   }: {
     model: RealtimeModel;
     chatCtx?: llm.ChatContext;
     fncCtx?: llm.FunctionContext;
+    maxTextResponseRetries?: number;
   }) {
     super();
     this.model = model;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.#maxTextResponseRetries = maxTextResponseRetries;
   }
 
   #participant: RemoteParticipant | string | null = null;
@@ -146,7 +153,7 @@ export class MultimodalAgent extends EventEmitter {
         if (this.linkedParticipant) {
           return;
         }
-        this.#linkParticipant(participant.identity);
+        this.#linkParticipant(participant.identity!);
       });
       room.on(
         RoomEvent.TrackPublished,
@@ -220,12 +227,12 @@ export class MultimodalAgent extends EventEmitter {
         if (typeof participant === 'string') {
           this.#linkParticipant(participant);
         } else {
-          this.#linkParticipant(participant.identity);
+          this.#linkParticipant(participant.identity!);
         }
       } else {
         // No participant specified, try to find the first participant in the room
         for (const participant of room.remoteParticipants.values()) {
-          this.#linkParticipant(participant.identity);
+          this.#linkParticipant(participant.identity!);
           break;
         }
       }
@@ -236,9 +243,11 @@ export class MultimodalAgent extends EventEmitter {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       this.#session.on('response_content_added', (message: any) => {
         // openai.realtime.RealtimeContent
+        if (message.contentType === 'text') return;
+
         const trFwd = new BasicTranscriptionForwarder(
           this.room!,
-          this.room!.localParticipant!.identity,
+          this.room!.localParticipant!.identity!,
           this.#getLocalTrackSid()!,
           message.responseId,
         );
@@ -253,6 +262,36 @@ export class MultimodalAgent extends EventEmitter {
         this.#playingHandle = handle;
       });
 
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('response_content_done', (message: any) => {
+        // openai.realtime.RealtimeContent
+        if (message.contentType === 'text') {
+          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {
+            throw new Error(
+              'The OpenAI Realtime API returned a text response ' +
+                `after ${this.#maxTextResponseRetries} retries. ` +
+                'Please try to reduce the number of text system or ' +
+                'assistant messages in the chat context.',
+            );
+          }
+
+          this.#textResponseRetries++;
+          this.#logger
+            .child({
+              itemId: message.itemId,
+              text: message.text,
+              retries: this.#textResponseRetries,
+            })
+            .warn(
+              'The OpenAI Realtime API returned a text response instead of audio. ' +
+                'Attempting to recover to audio mode...',
+            );
+          this.#session!.recoverFromTextResponse(message.itemId);
+        } else {
+          this.#textResponseRetries = 0;
+        }
+      });
+
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       this.#session.on('input_speech_committed', (ev: any) => {
         // openai.realtime.InputSpeechCommittedEvent
@@ -419,7 +458,7 @@ export class MultimodalAgent extends EventEmitter {
 
   #getLocalTrackSid(): string | null {
     if (!this.#localTrackSid && this.room && this.room.localParticipant) {
-      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);
+      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant!.identity!);
     }
     return this.#localTrackSid;
   }
@@ -470,7 +509,7 @@ export class MultimodalAgent extends EventEmitter {
 
   #setState(state: AgentState) {
     if (this.room?.isConnected && this.room.localParticipant) {
-      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+      const currentState = this.room.localParticipant.attributes![AGENT_STATE_ATTRIBUTE];
       if (currentState !== state) {
         this.room.localParticipant.setAttributes({
           [AGENT_STATE_ATTRIBUTE]: state,

package/src/pipeline/pipeline_agent.ts

@@ -355,7 +355,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       if (this.#participant) {
         return;
       }
-      this.#linkParticipant.call(this, participant.identity);
+      this.#linkParticipant.call(this, participant.identity!);
     });
 
     this.#room = room;
@@ -365,7 +365,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       if (typeof participant === 'string') {
         this.#linkParticipant(participant);
       } else {
-        this.#linkParticipant(participant.identity);
+        this.#linkParticipant(participant.identity!);
       }
     }
 
@@ -488,7 +488,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       }
     });
     this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
-      this.emit(VPAEvent.USER_STARTED_SPEAKING);
+      this.emit(VPAEvent.USER_STOPPED_SPEAKING);
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
     this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
@@ -971,6 +971,7 @@ class DeferredReplyValidation {
   #speaking = false;
   #endOfSpeechDelay: number;
   #finalTranscriptDelay: number;
+  #abort?: AbortController;
 
   constructor(validateFunc: () => Promise<void>, minEndpointingDelay: number) {
     this.#validateFunc = validateFunc;
@@ -997,10 +998,9 @@ class DeferredReplyValidation {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   onHumanStartOfSpeech(_: VADEvent) {
     this.#speaking = true;
-    // TODO(nbsp):
-    // if (this.validating) {
-    //   this.#validatingPromise.cancel()
-    // }
+    if (this.validating) {
+      this.#abort?.abort();
+    }
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -1011,7 +1011,7 @@ class DeferredReplyValidation {
     if (this.#lastFinalTranscript) {
       const delay = this.#endWithPunctuation()
         ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR
-        : 1;
+        : 1_000;
       this.#run(delay);
     }
   }
@@ -1031,13 +1031,19 @@ class DeferredReplyValidation {
   }
 
   #run(delay: number) {
-    const runTask = async (delay: number) => {
-      await new Promise((resolve) => setTimeout(resolve, delay));
-      this.#resetStates();
-      await this.#validateFunc();
+    const runTask = async (delay: number, signal: AbortSignal) => {
+      const timeout = setTimeout(() => {
+        this.#resetStates();
+        this.#validateFunc();
+      }, delay);
+      signal.addEventListener('abort', () => {
+        clearTimeout(timeout);
+      });
     };
 
+    this.#abort?.abort();
+    this.#abort = new AbortController();
     this.#validatingFuture = new Future();
-    this.#validatingPromise = runTask(delay);
+    this.#validatingPromise = runTask(delay, this.#abort.signal);
   }
 }

package/src/utils.ts

@@ -91,10 +91,18 @@ export class Queue<T> {
   }
 
   async get(): Promise<T> {
-    if (this.items.length === 0) {
-      await once(this.#events, 'put');
-    }
-    const item = this.items.shift()!;
+    const _get = async (): Promise<T> => {
+      if (this.items.length === 0) {
+        await once(this.#events, 'put');
+      }
+      let item = this.items.shift();
+      if (!item) {
+        item = await _get();
+      }
+      return item;
+    };
+
+    const item = _get();
     this.#events.emit('get');
     return item;
   }

package/src/worker.ts

@@ -322,15 +322,19 @@ export class Worker {
         try {
           await new Promise((resolve, reject) => {
             this.#session!.on('open', resolve);
-            this.#session!.on('error', (error) => reject(error));
-            this.#session!.on('close', (code) => reject(new Error(`WebSocket returned ${code}`)));
+            this.#session!.on('error', (error) => reject(error.message));
+            this.#session!.on('close', (code) => reject(`WebSocket returned ${code}`));
           });
 
           retries = 0;
           this.#logger.debug('connected to LiveKit server');
           this.#runWS(this.#session);
           return;
-        } catch (e) {
+        } catch (e: unknown) {
+          if (e instanceof Error || e instanceof ErrorEvent) {
+            e = e.message;
+          }
+
           if (this.#closed) return;
           if (retries >= this.#opts.maxRetry) {
             throw new WorkerError(