@livekit/agents 1.0.16 → 1.0.18

package/dist/voice/agent_activity.js

@@ -152,6 +152,11 @@ class AgentActivity {
         } catch (error) {
           this.logger.error(error, "failed to update the tools");
         }
+        if (!this.llm.capabilities.audioOutput && !this.tts && this.agentSession.output.audio) {
+          this.logger.error(
+            "audio output is enabled but RealtimeModel has no audio modality and no TTS is set. Either enable audio modality in the RealtimeModel or set a TTS model."
+          );
+        }
       } else if (this.llm instanceof LLM) {
         try {
           updateInstructions({
@@ -449,7 +454,9 @@ class AgentActivity {
     }
     if (this.stt && this.agentSession.options.minInterruptionWords > 0 && this.audioRecognition) {
       const text = this.audioRecognition.currentTranscript;
-      if (text && splitWords(text, true).length < this.agentSession.options.minInterruptionWords) {
+      const normalizedText = text ?? "";
+      const wordCount = splitWords(normalizedText, true).length;
+      if (wordCount < this.agentSession.options.minInterruptionWords) {
         return;
       }
     }
@@ -551,10 +558,19 @@ class AgentActivity {
       this.logger.warn({ user_input: info.newTranscript }, "skipping user input, task is draining");
       return true;
     }
-    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && this.agentSession.options.minInterruptionWords > 0 && info.newTranscript.split(" ").length < this.agentSession.options.minInterruptionWords) {
-      this.cancelPreemptiveGeneration();
-      this.logger.info("skipping user input, new_transcript is too short");
-      return false;
+    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && this.agentSession.options.minInterruptionWords > 0) {
+      const wordCount = splitWords(info.newTranscript, true).length;
+      if (wordCount < this.agentSession.options.minInterruptionWords) {
+        this.cancelPreemptiveGeneration();
+        this.logger.info(
+          {
+            wordCount,
+            minInterruptionWords: this.agentSession.options.minInterruptionWords
+          },
+          "skipping user input, word count below minimum interruption threshold"
+        );
+        return false;
+      }
     }
     const oldTask = this._userTurnCompletedTask;
     this._userTurnCompletedTask = this.createSpeechTask({
@@ -1197,7 +1213,22 @@ ${instructions}` : instructions,
             );
             break;
           }
-          const trNodeResult = await this.agent.transcriptionNode(msg.textStream, modelSettings);
+          const msgModalities = msg.modalities ? await msg.modalities : void 0;
+          let ttsTextInput = null;
+          let trTextInput;
+          if (msgModalities && !msgModalities.includes("audio") && this.tts) {
+            if (this.llm instanceof RealtimeModel && this.llm.capabilities.audioOutput) {
+              this.logger.warn(
+                "text response received from realtime API, falling back to use a TTS model."
+              );
+            }
+            const [_ttsTextInput, _trTextInput] = msg.textStream.tee();
+            ttsTextInput = _ttsTextInput;
+            trTextInput = _trTextInput;
+          } else {
+            trTextInput = msg.textStream;
+          }
+          const trNodeResult = await this.agent.transcriptionNode(trTextInput, modelSettings);
           let textOut = null;
           if (trNodeResult) {
             const [textForwardTask, _textOut] = performTextForwarding(
@@ -1210,28 +1241,44 @@ ${instructions}` : instructions,
           }
           let audioOut = null;
           if (audioOutput) {
-            const realtimeAudio = await this.agent.realtimeAudioOutputNode(
-              msg.audioStream,
-              modelSettings
-            );
-            if (realtimeAudio) {
+            let realtimeAudioResult = null;
+            if (ttsTextInput) {
+              const [ttsTask, ttsStream] = performTTSInference(
+                (...args) => this.agent.ttsNode(...args),
+                ttsTextInput,
+                modelSettings,
+                abortController
+              );
+              tasks.push(ttsTask);
+              realtimeAudioResult = ttsStream;
+            } else if (msgModalities && msgModalities.includes("audio")) {
+              realtimeAudioResult = await this.agent.realtimeAudioOutputNode(
+                msg.audioStream,
+                modelSettings
+              );
+            } else if (this.llm instanceof RealtimeModel && this.llm.capabilities.audioOutput) {
+              this.logger.error(
+                "Text message received from Realtime API with audio modality. This usually happens when text chat context is synced to the API. Try to add a TTS model as fallback or use text modality with TTS instead."
+              );
+            } else {
+              this.logger.warn(
+                "audio output is enabled but neither tts nor realtime audio is available"
+              );
+            }
+            if (realtimeAudioResult) {
               const [forwardTask, _audioOut] = performAudioForwarding(
-                realtimeAudio,
+                realtimeAudioResult,
                 audioOutput,
                 abortController
               );
               forwardTasks.push(forwardTask);
               audioOut = _audioOut;
               audioOut.firstFrameFut.await.finally(onFirstFrame);
-            } else {
-              this.logger.warn(
-                "audio output is enabled but neither tts nor realtime audio is available"
-              );
             }
           } else if (textOut) {
             textOut.firstTextFut.await.finally(onFirstFrame);
           }
-          outputs.push([msg.messageId, textOut, audioOut]);
+          outputs.push([msg.messageId, textOut, audioOut, msgModalities]);
         }
         await waitFor(forwardTasks);
       } catch (error) {
@@ -1301,7 +1348,7 @@ ${instructions}` : instructions,
       replyAbortController.abort();
       await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
       if (messageOutputs.length > 0) {
-        const [msgId, textOut, audioOut] = messageOutputs[0];
+        const [msgId, textOut, audioOut, msgModalities] = messageOutputs[0];
         let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
         if (audioOutput) {
           audioOutput.clearBuffer();
@@ -1321,7 +1368,9 @@ ${instructions}` : instructions,
           }
           this.realtimeSession.truncate({
             messageId: msgId,
-            audioEndMs: Math.floor(playbackPosition)
+            audioEndMs: Math.floor(playbackPosition),
+            modalities: msgModalities,
+            audioTranscript: forwardedText
           });
         }
         if (forwardedText) {
@@ -1345,7 +1394,7 @@ ${instructions}` : instructions,
       return;
     }
     if (messageOutputs.length > 0) {
-      const [msgId, textOut, _] = messageOutputs[0];
+      const [msgId, textOut, _, __] = messageOutputs[0];
       const message = ChatMessage.create({
         role: "assistant",
         content: (textOut == null ? void 0 : textOut.text) || "",