@livekit/agents 1.0.37 → 1.0.39

package/dist/voice/agent_activity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAqBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAA4C;IAE/D,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA6F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoH5B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAwB/D,gBAAgB,IAAI,IAAI;IAIxB,cAAc;IAUd,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAwDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAsCrD,eAAe,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI;IAIpC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IA+CtC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAiB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAiBxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IA4BlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAuCtB,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAuGhB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;IA0BzB,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IA0HrB,OAAO,CAAC,sBAAsB,CAyX5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;YAkc3B,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;IAiBhB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA2BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAyC7B"}
+{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAqBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAA4C;IAE/D,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA8F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoH5B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAwB/D,gBAAgB,IAAI,IAAI;IAIxB,cAAc;IAUd,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAwDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAqCrD,eAAe,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQnC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAWtC,OAAO,CAAC,wBAAwB;IAyChC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAqB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IA+BxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IA4BlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAuCtB,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAuGhB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;IA0BzB,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IAmIrB,OAAO,CAAC,sBAAsB,CAkZ5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;YAwc3B,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;IAiBhB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA2BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAyC7B"}

package/dist/voice/agent_activity.js

@@ -1,5 +1,5 @@
 import { Mutex } from "@livekit/mutex";
-import { ROOT_CONTEXT, trace } from "@opentelemetry/api";
+import { ROOT_CONTEXT, context as otelContext, trace } from "@opentelemetry/api";
 import { Heap } from "heap-js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { ReadableStream } from "node:stream/web";
@@ -119,9 +119,9 @@ class AgentActivity {
       );
       this.turnDetectionMode = void 0;
     }
-    if (!this.vad && this.stt && this.llm instanceof LLM && this.allowInterruptions && this.turnDetectionMode === void 0) {
+    if (!this.vad && this.stt && !this.stt.capabilities.streaming && this.llm instanceof LLM && this.allowInterruptions && this.turnDetectionMode === void 0) {
       this.logger.warn(
-        "VAD is not set. Enabling VAD is recommended when using LLM and STT for more responsive interruption handling."
+        "VAD is not set. Enabling VAD is recommended when using LLM and non-streaming STT for more responsive interruption handling."
       );
     }
   }
@@ -455,8 +455,12 @@ class AgentActivity {
     this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
   }
   // recognition hooks
-  onStartOfSpeech(_ev) {
-    this.agentSession._updateUserState("speaking");
+  onStartOfSpeech(ev) {
+    let speechStartTime = Date.now();
+    if (ev) {
+      speechStartTime = speechStartTime - ev.speechDuration;
+    }
+    this.agentSession._updateUserState("speaking", speechStartTime);
   }
   onEndOfSpeech(ev) {
     let speechEndTime = Date.now();
@@ -466,14 +470,16 @@ class AgentActivity {
     this.agentSession._updateUserState("listening", speechEndTime);
   }
   onVADInferenceDone(ev) {
-    var _a, _b;
     if (this.turnDetection === "manual" || this.turnDetection === "realtime_llm") {
       return;
     }
-    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
-      return;
+    if (ev.speechDuration >= this.agentSession.options.minInterruptionDuration) {
+      this.interruptByAudioActivity();
     }
-    if (ev.speechDuration < this.agentSession.options.minInterruptionDuration) {
+  }
+  interruptByAudioActivity() {
+    var _a, _b;
+    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
       return;
     }
     if (this.stt && this.agentSession.options.minInterruptionWords > 0 && this.audioRecognition) {
@@ -486,7 +492,10 @@ class AgentActivity {
     }
     (_a = this.realtimeSession) == null ? void 0 : _a.startUserActivity();
     if (this._currentSpeech && !this._currentSpeech.interrupted && this._currentSpeech.allowInterruptions) {
-      this.logger.info({ "speech id": this._currentSpeech.id }, "speech interrupted by VAD");
+      this.logger.info(
+        { "speech id": this._currentSpeech.id },
+        "speech interrupted by audio activity"
+      );
       (_b = this.realtimeSession) == null ? void 0 : _b.interrupt();
       this._currentSpeech.interrupt();
     }
@@ -504,6 +513,9 @@ class AgentActivity {
         // TODO(AJS-106): add multi participant support
       })
     );
+    if (ev.alternatives[0].text) {
+      this.interruptByAudioActivity();
+    }
   }
   onFinalTranscript(ev) {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
@@ -518,6 +530,9 @@ class AgentActivity {
         // TODO(AJS-106): add multi participant support
       })
     );
+    if (this.audioRecognition && this.turnDetection !== "manual" && this.turnDetection !== "realtime_llm") {
+      this.interruptByAudioActivity();
+    }
   }
   onPreemptiveGeneration(info) {
     if (!this.agentSession.options.preemptiveGeneration || this.draining || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof LLM)) {
@@ -833,6 +848,7 @@ ${instructions}` : instructions,
     );
   }
   async ttsTask(speechHandle, text, addToChatCtx, modelSettings, replyAbortController, audio) {
+    speechHandle._agentTurnContext = otelContext.active();
     speechHandleStorage.enterWith(speechHandle);
     const transcriptionOutput = this.agentSession.output.transcriptionEnabled ? this.agentSession.output.transcription : null;
     const audioOutput = this.agentSession.output.audioEnabled ? this.agentSession.output.audio : null;
@@ -864,12 +880,15 @@ ${instructions}` : instructions,
       textOut = _textOut;
       tasks.push(textForwardTask);
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     if (!audioOutput) {
       if (textOut) {
-        textOut.firstTextFut.await.finally(onFirstFrame);
+        textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
       }
     } else {
       let audioOut = null;
@@ -897,7 +916,7 @@ ${instructions}` : instructions,
         tasks.push(forwardTask);
         audioOut = _audioOut;
       }
-      audioOut.firstFrameFut.await.finally(onFirstFrame);
+      audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
     }
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
@@ -936,6 +955,7 @@ ${instructions}` : instructions,
     span
   }) => {
     var _a, _b, _c;
+    speechHandle._agentTurnContext = otelContext.active();
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     if (instructions) {
       span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, instructions);
@@ -1012,8 +1032,11 @@ ${instructions}` : instructions,
       tasks.push(textForwardTask);
       textOut = _textOut;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     let audioOut = null;
     if (audioOutput) {
@@ -1025,12 +1048,12 @@ ${instructions}` : instructions,
         );
         audioOut = _audioOut;
         tasks.push(forwardTask);
-        audioOut.firstFrameFut.await.finally(onFirstFrame);
+        audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
       } else {
         throw Error("ttsStream is null when audioOutput is enabled");
       }
     } else {
-      textOut == null ? void 0 : textOut.firstTextFut.await.finally(onFirstFrame);
+      textOut == null ? void 0 : textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
     }
     const onToolExecutionStarted = (f) => {
       speechHandle._itemAdded([f]);
@@ -1061,7 +1084,12 @@ ${instructions}` : instructions,
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolsMessages);
-      this.agentSession._toolItemsAdded(toolsMessages);
+      const toolCallOutputs = toolsMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
@@ -1078,9 +1106,9 @@ ${instructions}` : instructions,
       let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
       if (audioOutput) {
         const playbackEv = await audioOutput.waitForPlayout();
-        if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+        if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
           this.logger.info(
-            { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+            { speech_id: speechHandle.id, playbackPositionInS: playbackEv.playbackPosition },
             "playout interrupted"
           );
           if (playbackEv.synchronizedTranscript) {
@@ -1218,7 +1246,12 @@ ${instructions}` : instructions,
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolMessages);
-      this.agentSession._toolItemsAdded(toolMessages);
+      const toolCallOutputs = toolMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
   };
   pipelineReplyTask = async (speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages) => tracer.startActiveSpan(
@@ -1261,6 +1294,7 @@ ${instructions}` : instructions,
     span
   }) {
     var _a, _b, _c;
+    speechHandle._agentTurnContext = otelContext.active();
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     speechHandleStorage.enterWith(speechHandle);
     if (!this.realtimeSession) {
@@ -1285,8 +1319,11 @@ ${instructions}` : instructions,
     if (speechHandle.interrupted) {
       return;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     const readMessages = async (abortController, outputs) => {
       replyAbortController.signal.addEventListener("abort", () => abortController.abort(), {
@@ -1361,10 +1398,10 @@ ${instructions}` : instructions,
               );
               forwardTasks.push(forwardTask);
               audioOut = _audioOut;
-              audioOut.firstFrameFut.await.finally(onFirstFrame);
+              audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
             }
           } else if (textOut) {
-            textOut.firstTextFut.await.finally(onFirstFrame);
+            textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
           }
           outputs.push([msg.messageId, textOut, audioOut, msgModalities]);
         }
@@ -1428,7 +1465,6 @@ ${instructions}` : instructions,
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
-      this.agentSession._updateAgentState("listening");
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
@@ -1443,10 +1479,10 @@ ${instructions}` : instructions,
         if (audioOutput) {
           audioOutput.clearBuffer();
           const playbackEv = await audioOutput.waitForPlayout();
-          let playbackPosition = playbackEv.playbackPosition;
-          if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+          let playbackPositionInS = playbackEv.playbackPosition;
+          if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
             this.logger.info(
-              { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+              { speech_id: speechHandle.id, playbackPositionInS },
               "playout interrupted"
             );
             if (playbackEv.synchronizedTranscript) {
@@ -1454,11 +1490,11 @@ ${instructions}` : instructions,
             }
           } else {
             forwardedText = "";
-            playbackPosition = 0;
+            playbackPositionInS = 0;
           }
           this.realtimeSession.truncate({
             messageId: msgId,
-            audioEndMs: Math.floor(playbackPosition),
+            audioEndMs: Math.floor(playbackPositionInS * 1e3),
             modalities: msgModalities,
             audioTranscript: forwardedText
           });
@@ -1496,14 +1532,13 @@ ${instructions}` : instructions,
       this.agentSession._conversationItemAdded(message);
     }
     speechHandle._markGenerationDone();
-    toolOutput.firstToolStartedFuture.await.finally(() => {
-      this.agentSession._updateAgentState("thinking");
-    });
     await executeToolsTask.result;
+    if (toolOutput.output.length > 0) {
+      this.agentSession._updateAgentState("thinking");
+    } else if (this.agentSession.agentState === "speaking") {
+      this.agentSession._updateAgentState("listening");
+    }
     if (toolOutput.output.length === 0) {
-      if (!speechHandle.interrupted) {
-        this.agentSession._updateAgentState("listening");
-      }
       return;
     }
     const { maxToolSteps } = this.agentSession.options;