@livekit/agents 1.0.37 → 1.0.38

package/dist/voice/agent_activity.js

@@ -1,5 +1,5 @@
 import { Mutex } from "@livekit/mutex";
-import { ROOT_CONTEXT, trace } from "@opentelemetry/api";
+import { ROOT_CONTEXT, context as otelContext, trace } from "@opentelemetry/api";
 import { Heap } from "heap-js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { ReadableStream } from "node:stream/web";
@@ -455,8 +455,12 @@ class AgentActivity {
     this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
   }
   // recognition hooks
-  onStartOfSpeech(_ev) {
-    this.agentSession._updateUserState("speaking");
+  onStartOfSpeech(ev) {
+    let speechStartTime = Date.now();
+    if (ev) {
+      speechStartTime = speechStartTime - ev.speechDuration;
+    }
+    this.agentSession._updateUserState("speaking", speechStartTime);
   }
   onEndOfSpeech(ev) {
     let speechEndTime = Date.now();
@@ -833,6 +837,7 @@ ${instructions}` : instructions,
     );
   }
   async ttsTask(speechHandle, text, addToChatCtx, modelSettings, replyAbortController, audio) {
+    speechHandle._agentTurnContext = otelContext.active();
     speechHandleStorage.enterWith(speechHandle);
     const transcriptionOutput = this.agentSession.output.transcriptionEnabled ? this.agentSession.output.transcription : null;
     const audioOutput = this.agentSession.output.audioEnabled ? this.agentSession.output.audio : null;
@@ -864,12 +869,15 @@ ${instructions}` : instructions,
       textOut = _textOut;
       tasks.push(textForwardTask);
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     if (!audioOutput) {
       if (textOut) {
-        textOut.firstTextFut.await.finally(onFirstFrame);
+        textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
       }
     } else {
       let audioOut = null;
@@ -897,7 +905,7 @@ ${instructions}` : instructions,
         tasks.push(forwardTask);
         audioOut = _audioOut;
       }
-      audioOut.firstFrameFut.await.finally(onFirstFrame);
+      audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
     }
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
@@ -936,6 +944,7 @@ ${instructions}` : instructions,
     span
   }) => {
     var _a, _b, _c;
+    speechHandle._agentTurnContext = otelContext.active();
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     if (instructions) {
       span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, instructions);
@@ -1012,8 +1021,11 @@ ${instructions}` : instructions,
       tasks.push(textForwardTask);
       textOut = _textOut;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     let audioOut = null;
     if (audioOutput) {
@@ -1025,12 +1037,12 @@ ${instructions}` : instructions,
         );
         audioOut = _audioOut;
         tasks.push(forwardTask);
-        audioOut.firstFrameFut.await.finally(onFirstFrame);
+        audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
       } else {
         throw Error("ttsStream is null when audioOutput is enabled");
       }
     } else {
-      textOut == null ? void 0 : textOut.firstTextFut.await.finally(onFirstFrame);
+      textOut == null ? void 0 : textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
     }
     const onToolExecutionStarted = (f) => {
       speechHandle._itemAdded([f]);
@@ -1061,7 +1073,12 @@ ${instructions}` : instructions,
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolsMessages);
-      this.agentSession._toolItemsAdded(toolsMessages);
+      const toolCallOutputs = toolsMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
@@ -1078,9 +1095,9 @@ ${instructions}` : instructions,
       let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
       if (audioOutput) {
         const playbackEv = await audioOutput.waitForPlayout();
-        if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+        if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
           this.logger.info(
-            { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+            { speech_id: speechHandle.id, playbackPositionInS: playbackEv.playbackPosition },
             "playout interrupted"
           );
           if (playbackEv.synchronizedTranscript) {
@@ -1218,7 +1235,12 @@ ${instructions}` : instructions,
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolMessages);
-      this.agentSession._toolItemsAdded(toolMessages);
+      const toolCallOutputs = toolMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
   };
   pipelineReplyTask = async (speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages) => tracer.startActiveSpan(
@@ -1261,6 +1283,7 @@ ${instructions}` : instructions,
     span
   }) {
     var _a, _b, _c;
+    speechHandle._agentTurnContext = otelContext.active();
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     speechHandleStorage.enterWith(speechHandle);
     if (!this.realtimeSession) {
@@ -1285,8 +1308,11 @@ ${instructions}` : instructions,
     if (speechHandle.interrupted) {
       return;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     const readMessages = async (abortController, outputs) => {
       replyAbortController.signal.addEventListener("abort", () => abortController.abort(), {
@@ -1361,10 +1387,10 @@ ${instructions}` : instructions,
               );
               forwardTasks.push(forwardTask);
               audioOut = _audioOut;
-              audioOut.firstFrameFut.await.finally(onFirstFrame);
+              audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
             }
           } else if (textOut) {
-            textOut.firstTextFut.await.finally(onFirstFrame);
+            textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
           }
           outputs.push([msg.messageId, textOut, audioOut, msgModalities]);
         }
@@ -1443,10 +1469,10 @@ ${instructions}` : instructions,
         if (audioOutput) {
           audioOutput.clearBuffer();
           const playbackEv = await audioOutput.waitForPlayout();
-          let playbackPosition = playbackEv.playbackPosition;
-          if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+          let playbackPositionInS = playbackEv.playbackPosition;
+          if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
             this.logger.info(
-              { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+              { speech_id: speechHandle.id, playbackPositionInS },
               "playout interrupted"
             );
             if (playbackEv.synchronizedTranscript) {
@@ -1454,11 +1480,11 @@ ${instructions}` : instructions,
             }
           } else {
             forwardedText = "";
-            playbackPosition = 0;
+            playbackPositionInS = 0;
           }
           this.realtimeSession.truncate({
             messageId: msgId,
-            audioEndMs: Math.floor(playbackPosition),
+            audioEndMs: Math.floor(playbackPositionInS * 1e3),
             modalities: msgModalities,
             audioTranscript: forwardedText
           });