@livekit/agents 1.0.37 → 1.0.39

package/dist/voice/agent_activity.cjs

@@ -122,9 +122,9 @@ class AgentActivity {
       );
       this.turnDetectionMode = void 0;
     }
-    if (!this.vad && this.stt && this.llm instanceof import_llm.LLM && this.allowInterruptions && this.turnDetectionMode === void 0) {
+    if (!this.vad && this.stt && !this.stt.capabilities.streaming && this.llm instanceof import_llm.LLM && this.allowInterruptions && this.turnDetectionMode === void 0) {
       this.logger.warn(
-        "VAD is not set. Enabling VAD is recommended when using LLM and STT for more responsive interruption handling."
+        "VAD is not set. Enabling VAD is recommended when using LLM and non-streaming STT for more responsive interruption handling."
       );
     }
   }
@@ -458,8 +458,12 @@ class AgentActivity {
     this.scheduleSpeech(handle, import_speech_handle.SpeechHandle.SPEECH_PRIORITY_NORMAL);
   }
   // recognition hooks
-  onStartOfSpeech(_ev) {
-    this.agentSession._updateUserState("speaking");
+  onStartOfSpeech(ev) {
+    let speechStartTime = Date.now();
+    if (ev) {
+      speechStartTime = speechStartTime - ev.speechDuration;
+    }
+    this.agentSession._updateUserState("speaking", speechStartTime);
   }
   onEndOfSpeech(ev) {
     let speechEndTime = Date.now();
@@ -469,14 +473,16 @@ class AgentActivity {
     this.agentSession._updateUserState("listening", speechEndTime);
   }
   onVADInferenceDone(ev) {
-    var _a, _b;
     if (this.turnDetection === "manual" || this.turnDetection === "realtime_llm") {
       return;
     }
-    if (this.llm instanceof import_llm.RealtimeModel && this.llm.capabilities.turnDetection) {
-      return;
+    if (ev.speechDuration >= this.agentSession.options.minInterruptionDuration) {
+      this.interruptByAudioActivity();
     }
-    if (ev.speechDuration < this.agentSession.options.minInterruptionDuration) {
+  }
+  interruptByAudioActivity() {
+    var _a, _b;
+    if (this.llm instanceof import_llm.RealtimeModel && this.llm.capabilities.turnDetection) {
       return;
     }
     if (this.stt && this.agentSession.options.minInterruptionWords > 0 && this.audioRecognition) {
@@ -489,7 +495,10 @@ class AgentActivity {
     }
     (_a = this.realtimeSession) == null ? void 0 : _a.startUserActivity();
     if (this._currentSpeech && !this._currentSpeech.interrupted && this._currentSpeech.allowInterruptions) {
-      this.logger.info({ "speech id": this._currentSpeech.id }, "speech interrupted by VAD");
+      this.logger.info(
+        { "speech id": this._currentSpeech.id },
+        "speech interrupted by audio activity"
+      );
       (_b = this.realtimeSession) == null ? void 0 : _b.interrupt();
       this._currentSpeech.interrupt();
     }
@@ -507,6 +516,9 @@ class AgentActivity {
         // TODO(AJS-106): add multi participant support
       })
     );
+    if (ev.alternatives[0].text) {
+      this.interruptByAudioActivity();
+    }
   }
   onFinalTranscript(ev) {
     if (this.llm instanceof import_llm.RealtimeModel && this.llm.capabilities.userTranscription) {
@@ -521,6 +533,9 @@ class AgentActivity {
         // TODO(AJS-106): add multi participant support
       })
     );
+    if (this.audioRecognition && this.turnDetection !== "manual" && this.turnDetection !== "realtime_llm") {
+      this.interruptByAudioActivity();
+    }
   }
   onPreemptiveGeneration(info) {
     if (!this.agentSession.options.preemptiveGeneration || this.draining || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof import_llm.LLM)) {
@@ -836,6 +851,7 @@ ${instructions}` : instructions,
     );
   }
   async ttsTask(speechHandle, text, addToChatCtx, modelSettings, replyAbortController, audio) {
+    speechHandle._agentTurnContext = import_api.context.active();
     speechHandleStorage.enterWith(speechHandle);
     const transcriptionOutput = this.agentSession.output.transcriptionEnabled ? this.agentSession.output.transcription : null;
     const audioOutput = this.agentSession.output.audioEnabled ? this.agentSession.output.audio : null;
@@ -867,12 +883,15 @@ ${instructions}` : instructions,
       textOut = _textOut;
       tasks.push(textForwardTask);
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     if (!audioOutput) {
       if (textOut) {
-        textOut.firstTextFut.await.finally(onFirstFrame);
+        textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
       }
     } else {
       let audioOut = null;
@@ -900,7 +919,7 @@ ${instructions}` : instructions,
         tasks.push(forwardTask);
         audioOut = _audioOut;
       }
-      audioOut.firstFrameFut.await.finally(onFirstFrame);
+      audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
     }
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
@@ -939,6 +958,7 @@ ${instructions}` : instructions,
     span
   }) => {
     var _a, _b, _c;
+    speechHandle._agentTurnContext = import_api.context.active();
     span.setAttribute(import_telemetry.traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     if (instructions) {
       span.setAttribute(import_telemetry.traceTypes.ATTR_INSTRUCTIONS, instructions);
@@ -1015,8 +1035,11 @@ ${instructions}` : instructions,
       tasks.push(textForwardTask);
       textOut = _textOut;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     let audioOut = null;
     if (audioOutput) {
@@ -1028,12 +1051,12 @@ ${instructions}` : instructions,
         );
         audioOut = _audioOut;
         tasks.push(forwardTask);
-        audioOut.firstFrameFut.await.finally(onFirstFrame);
+        audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
       } else {
         throw Error("ttsStream is null when audioOutput is enabled");
       }
     } else {
-      textOut == null ? void 0 : textOut.firstTextFut.await.finally(onFirstFrame);
+      textOut == null ? void 0 : textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
     }
     const onToolExecutionStarted = (f) => {
       speechHandle._itemAdded([f]);
@@ -1064,7 +1087,12 @@ ${instructions}` : instructions,
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolsMessages);
-      this.agentSession._toolItemsAdded(toolsMessages);
+      const toolCallOutputs = toolsMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
@@ -1081,9 +1109,9 @@ ${instructions}` : instructions,
       let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
       if (audioOutput) {
         const playbackEv = await audioOutput.waitForPlayout();
-        if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+        if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
           this.logger.info(
-            { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+            { speech_id: speechHandle.id, playbackPositionInS: playbackEv.playbackPosition },
             "playout interrupted"
           );
           if (playbackEv.synchronizedTranscript) {
@@ -1221,7 +1249,12 @@ ${instructions}` : instructions,
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolMessages);
-      this.agentSession._toolItemsAdded(toolMessages);
+      const toolCallOutputs = toolMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
   };
   pipelineReplyTask = async (speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages) => import_telemetry.tracer.startActiveSpan(
@@ -1264,6 +1297,7 @@ ${instructions}` : instructions,
     span
   }) {
     var _a, _b, _c;
+    speechHandle._agentTurnContext = import_api.context.active();
     span.setAttribute(import_telemetry.traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     speechHandleStorage.enterWith(speechHandle);
     if (!this.realtimeSession) {
@@ -1288,8 +1322,11 @@ ${instructions}` : instructions,
     if (speechHandle.interrupted) {
       return;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     const readMessages = async (abortController, outputs) => {
       replyAbortController.signal.addEventListener("abort", () => abortController.abort(), {
@@ -1364,10 +1401,10 @@ ${instructions}` : instructions,
               );
               forwardTasks.push(forwardTask);
               audioOut = _audioOut;
-              audioOut.firstFrameFut.await.finally(onFirstFrame);
+              audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
             }
           } else if (textOut) {
-            textOut.firstTextFut.await.finally(onFirstFrame);
+            textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
           }
           outputs.push([msg.messageId, textOut, audioOut, msgModalities]);
         }
@@ -1431,7 +1468,6 @@ ${instructions}` : instructions,
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
-      this.agentSession._updateAgentState("listening");
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
@@ -1446,10 +1482,10 @@ ${instructions}` : instructions,
         if (audioOutput) {
           audioOutput.clearBuffer();
           const playbackEv = await audioOutput.waitForPlayout();
-          let playbackPosition = playbackEv.playbackPosition;
-          if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+          let playbackPositionInS = playbackEv.playbackPosition;
+          if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
             this.logger.info(
-              { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+              { speech_id: speechHandle.id, playbackPositionInS },
               "playout interrupted"
             );
             if (playbackEv.synchronizedTranscript) {
@@ -1457,11 +1493,11 @@ ${instructions}` : instructions,
             }
           } else {
             forwardedText = "";
-            playbackPosition = 0;
+            playbackPositionInS = 0;
           }
           this.realtimeSession.truncate({
             messageId: msgId,
-            audioEndMs: Math.floor(playbackPosition),
+            audioEndMs: Math.floor(playbackPositionInS * 1e3),
             modalities: msgModalities,
             audioTranscript: forwardedText
           });
@@ -1499,14 +1535,13 @@ ${instructions}` : instructions,
       this.agentSession._conversationItemAdded(message);
     }
     speechHandle._markGenerationDone();
-    toolOutput.firstToolStartedFuture.await.finally(() => {
-      this.agentSession._updateAgentState("thinking");
-    });
     await executeToolsTask.result;
+    if (toolOutput.output.length > 0) {
+      this.agentSession._updateAgentState("thinking");
+    } else if (this.agentSession.agentState === "speaking") {
+      this.agentSession._updateAgentState("listening");
+    }
     if (toolOutput.output.length === 0) {
-      if (!speechHandle.interrupted) {
-        this.agentSession._updateAgentState("listening");
-      }
       return;
     }
     const { maxToolSteps } = this.agentSession.options;