@livekit/agents 0.6.0 → 0.6.1

package/src/multimodal/multimodal_agent.ts

@@ -36,6 +36,7 @@ export abstract class RealtimeSession extends EventEmitter {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer
   abstract fncCtx: llm.FunctionContext | undefined;
+  abstract recoverFromTextResponse(itemId: string): void;
 }
 
 /**
@@ -63,19 +64,25 @@ export class MultimodalAgent extends EventEmitter {
   subscribedTrack: RemoteAudioTrack | null = null;
   readMicroTask: Promise<void> | null = null;
 
+  #textResponseRetries = 0;
+  #maxTextResponseRetries: number;
+
   constructor({
     model,
     chatCtx,
     fncCtx,
+    maxTextResponseRetries = 5,
   }: {
     model: RealtimeModel;
     chatCtx?: llm.ChatContext;
     fncCtx?: llm.FunctionContext;
+    maxTextResponseRetries?: number;
   }) {
     super();
     this.model = model;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.#maxTextResponseRetries = maxTextResponseRetries;
   }
 
   #participant: RemoteParticipant | string | null = null;
@@ -146,7 +153,7 @@ export class MultimodalAgent extends EventEmitter {
         if (this.linkedParticipant) {
           return;
         }
-        this.#linkParticipant(participant.identity);
+        this.#linkParticipant(participant.identity!);
       });
       room.on(
         RoomEvent.TrackPublished,
@@ -220,12 +227,12 @@ export class MultimodalAgent extends EventEmitter {
         if (typeof participant === 'string') {
           this.#linkParticipant(participant);
         } else {
-          this.#linkParticipant(participant.identity);
+          this.#linkParticipant(participant.identity!);
         }
       } else {
         // No participant specified, try to find the first participant in the room
         for (const participant of room.remoteParticipants.values()) {
-          this.#linkParticipant(participant.identity);
+          this.#linkParticipant(participant.identity!);
           break;
         }
       }
@@ -236,9 +243,11 @@ export class MultimodalAgent extends EventEmitter {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       this.#session.on('response_content_added', (message: any) => {
         // openai.realtime.RealtimeContent
+        if (message.contentType === 'text') return;
+
         const trFwd = new BasicTranscriptionForwarder(
           this.room!,
-          this.room!.localParticipant!.identity,
+          this.room!.localParticipant!.identity!,
           this.#getLocalTrackSid()!,
           message.responseId,
         );
@@ -253,6 +262,36 @@ export class MultimodalAgent extends EventEmitter {
         this.#playingHandle = handle;
       });
 
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('response_content_done', (message: any) => {
+        // openai.realtime.RealtimeContent
+        if (message.contentType === 'text') {
+          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {
+            throw new Error(
+              'The OpenAI Realtime API returned a text response ' +
+                `after ${this.#maxTextResponseRetries} retries. ` +
+                'Please try to reduce the number of text system or ' +
+                'assistant messages in the chat context.',
+            );
+          }
+
+          this.#textResponseRetries++;
+          this.#logger
+            .child({
+              itemId: message.itemId,
+              text: message.text,
+              retries: this.#textResponseRetries,
+            })
+            .warn(
+              'The OpenAI Realtime API returned a text response instead of audio. ' +
+                'Attempting to recover to audio mode...',
+            );
+          this.#session!.recoverFromTextResponse(message.itemId);
+        } else {
+          this.#textResponseRetries = 0;
+        }
+      });
+
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       this.#session.on('input_speech_committed', (ev: any) => {
         // openai.realtime.InputSpeechCommittedEvent
@@ -419,7 +458,7 @@ export class MultimodalAgent extends EventEmitter {
 
   #getLocalTrackSid(): string | null {
     if (!this.#localTrackSid && this.room && this.room.localParticipant) {
-      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);
+      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant!.identity!);
     }
     return this.#localTrackSid;
   }
@@ -470,7 +509,7 @@ export class MultimodalAgent extends EventEmitter {
 
   #setState(state: AgentState) {
     if (this.room?.isConnected && this.room.localParticipant) {
-      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+      const currentState = this.room.localParticipant.attributes![AGENT_STATE_ATTRIBUTE];
       if (currentState !== state) {
         this.room.localParticipant.setAttributes({
           [AGENT_STATE_ATTRIBUTE]: state,

package/src/pipeline/pipeline_agent.ts

@@ -355,7 +355,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       if (this.#participant) {
         return;
       }
-      this.#linkParticipant.call(this, participant.identity);
+      this.#linkParticipant.call(this, participant.identity!);
     });
 
     this.#room = room;
@@ -365,7 +365,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       if (typeof participant === 'string') {
         this.#linkParticipant(participant);
       } else {
-        this.#linkParticipant(participant.identity);
+        this.#linkParticipant(participant.identity!);
       }
     }
 

package/src/utils.ts

@@ -91,10 +91,18 @@ export class Queue<T> {
   }
 
   async get(): Promise<T> {
-    if (this.items.length === 0) {
-      await once(this.#events, 'put');
-    }
-    const item = this.items.shift()!;
+    const _get = async (): Promise<T> => {
+      if (this.items.length === 0) {
+        await once(this.#events, 'put');
+      }
+      let item = this.items.shift();
+      if (!item) {
+        item = await _get();
+      }
+      return item;
+    };
+
+    const item = _get();
     this.#events.emit('get');
     return item;
   }

package/src/worker.ts

@@ -322,15 +322,19 @@ export class Worker {
         try {
           await new Promise((resolve, reject) => {
             this.#session!.on('open', resolve);
-            this.#session!.on('error', (error) => reject(error));
-            this.#session!.on('close', (code) => reject(new Error(`WebSocket returned ${code}`)));
+            this.#session!.on('error', (error) => reject(error.message));
+            this.#session!.on('close', (code) => reject(`WebSocket returned ${code}`));
           });
 
           retries = 0;
           this.#logger.debug('connected to LiveKit server');
           this.#runWS(this.#session);
           return;
-        } catch (e) {
+        } catch (e: unknown) {
+          if (e instanceof Error || e instanceof ErrorEvent) {
+            e = e.message;
+          }
+
           if (this.#closed) return;
           if (retries >= this.#opts.maxRetry) {
             throw new WorkerError(