@livekit/agents 1.0.47 → 1.1.0-dev.0

package/dist/voice/agent_activity.js

@@ -2,11 +2,13 @@ import { Mutex } from "@livekit/mutex";
 import { ROOT_CONTEXT, context as otelContext, trace } from "@opentelemetry/api";
 import { Heap } from "heap-js";
 import { AsyncLocalStorage } from "node:async_hooks";
-import { ReadableStream } from "node:stream/web";
+import { ReadableStream, TransformStream } from "node:stream/web";
+import { AdaptiveInterruptionDetector } from "../inference/interruption/interruption_detector.js";
 import { ChatMessage } from "../llm/chat_context.js";
 import {
   LLM,
-  RealtimeModel
+  RealtimeModel,
+  ToolFlag
 } from "../llm/index.js";
 import { isSameToolChoice, isSameToolContext } from "../llm/tool_context.js";
 import { log } from "../log.js";
@@ -48,6 +50,7 @@ import {
 import { SpeechHandle } from "./speech_handle.js";
 import { setParticipantSpanAttributes } from "./utils.js";
 const agentActivityStorage = new AsyncLocalStorage();
+const onEnterStorage = new AsyncLocalStorage();
 class AgentActivity {
   agent;
   agentSession;
@@ -72,16 +75,34 @@ class AgentActivity {
   // default to null as None, which maps to the default provider tool choice value
   toolChoice = null;
   _preemptiveGeneration;
-  /** @internal */
-  _mainTask;
-  _onEnterTask;
-  _onExitTask;
-  _userTurnCompletedTask;
+  interruptionDetector;
+  isInterruptionDetectionEnabled;
+  isInterruptionByAudioActivityEnabled;
+  isDefaultInterruptionByAudioActivityEnabled;
   onRealtimeGenerationCreated = (ev) => this.onGenerationCreated(ev);
   onRealtimeInputSpeechStarted = (ev) => this.onInputSpeechStarted(ev);
   onRealtimeInputSpeechStopped = (ev) => this.onInputSpeechStopped(ev);
   onRealtimeInputAudioTranscriptionCompleted = (ev) => this.onInputAudioTranscriptionCompleted(ev);
   onModelError = (ev) => this.onError(ev);
+  onInterruptionOverlappingSpeech = (ev) => {
+    this.agentSession.emit(AgentSessionEventTypes.UserOverlappingSpeech, ev);
+  };
+  onInterruptionMetricsCollected = (ev) => {
+    this.agentSession.emit(
+      AgentSessionEventTypes.MetricsCollected,
+      createMetricsCollectedEvent({ metrics: ev })
+    );
+  };
+  onInterruptionError = (ev) => {
+    const errorEvent = createErrorEvent(ev, this.interruptionDetector);
+    this.agentSession.emit(AgentSessionEventTypes.Error, errorEvent);
+    this.agentSession._onError(ev);
+  };
+  /** @internal */
+  _mainTask;
+  _onEnterTask;
+  _onExitTask;
+  _userTurnCompletedTask;
   constructor(agent, agentSession) {
     this.agent = agent;
     this.agentSession = agentSession;
@@ -140,6 +161,10 @@ class AgentActivity {
         "VAD is not set. Enabling VAD is recommended when using LLM and non-streaming STT for more responsive interruption handling."
       );
     }
+    this.interruptionDetector = this.resolveInterruptionDetector();
+    this.isInterruptionDetectionEnabled = !!this.interruptionDetector;
+    this.isInterruptionByAudioActivityEnabled = this.turnDetectionMode !== "manual" && this.turnDetectionMode !== "realtime_llm";
+    this.isDefaultInterruptionByAudioActivityEnabled = this.isInterruptionByAudioActivityEnabled;
   }
   async start() {
     const unlock = await this.lock.lock();
@@ -232,8 +257,9 @@ class AgentActivity {
       vad: this.vad,
       turnDetector: typeof this.turnDetection === "string" ? void 0 : this.turnDetection,
       turnDetectionMode: this.turnDetectionMode,
-      minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
-      maxEndpointingDelay: this.agentSession.options.maxEndpointingDelay,
+      interruptionDetection: this.interruptionDetector,
+      minEndpointingDelay: this.agentSession.options.turnHandling.endpointing.minDelay,
+      maxEndpointingDelay: this.agentSession.options.turnHandling.endpointing.maxDelay,
       rootSpanContext: this.agentSession.rootSpanContext,
       sttModel: (_a = this.stt) == null ? void 0 : _a.label,
       sttProvider: this.getSttProvider(),
@@ -247,11 +273,14 @@ class AgentActivity {
     this._resumeSchedulingTask();
     if (runOnEnter) {
       this._onEnterTask = this.createSpeechTask({
-        taskFn: () => tracer.startActiveSpan(async () => this.agent.onEnter(), {
-          name: "on_enter",
-          context: trace.setSpan(ROOT_CONTEXT, startSpan),
-          attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id }
-        }),
+        taskFn: () => onEnterStorage.run(
+          { session: this.agentSession, agent: this.agent },
+          () => tracer.startActiveSpan(async () => this.agent.onEnter(), {
+            name: "on_enter",
+            context: trace.setSpan(ROOT_CONTEXT, startSpan),
+            attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id }
+          })
+        ),
         inlineTask: true,
         name: "AgentActivity_onEnter"
       });
@@ -292,7 +321,8 @@ class AgentActivity {
     return this.realtimeSession;
   }
   get allowInterruptions() {
-    return this.agentSession.options.allowInterruptions;
+    var _a;
+    return ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.mode) !== false;
   }
   get useTtsAlignedTranscript() {
     return this.agent.useTtsAlignedTranscript ?? this.agentSession.useTtsAlignedTranscript;
@@ -303,6 +333,11 @@ class AgentActivity {
   get toolCtx() {
     return this.agent.toolCtx;
   }
+  /** @internal */
+  get inputStartedAt() {
+    var _a;
+    return (_a = this.audioRecognition) == null ? void 0 : _a.inputStartedAt;
+  }
   async updateChatCtx(chatCtx) {
     chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });
     this.agent._chatCtx = chatCtx;
@@ -317,19 +352,50 @@ class AgentActivity {
       });
     }
   }
-  updateOptions({ toolChoice }) {
+  // TODO: Add when AgentConfigUpdate is ported to ChatContext.
+  async updateTools(tools) {
+    this.agent._tools = { ...tools };
+    if (this.realtimeSession) {
+      await this.realtimeSession.updateTools(tools);
+    }
+    if (this.llm instanceof LLM) {
+      await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: tools }));
+    }
+  }
+  updateOptions({
+    toolChoice,
+    turnDetection
+  }) {
     if (toolChoice !== void 0) {
       this.toolChoice = toolChoice;
     }
     if (this.realtimeSession) {
       this.realtimeSession.updateOptions({ toolChoice: this.toolChoice });
     }
+    if (turnDetection !== void 0) {
+      this.turnDetectionMode = turnDetection;
+      this.isDefaultInterruptionByAudioActivityEnabled = this.turnDetectionMode !== "manual" && this.turnDetectionMode !== "realtime_llm";
+      if (this.agentSession.agentState !== "speaking") {
+        this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
+      }
+    }
+    if (this.audioRecognition) {
+      this.audioRecognition.updateOptions({ turnDetection: this.turnDetectionMode });
+    }
   }
   attachAudioInput(audioStream) {
     void this.audioStream.close();
     this.audioStream = new MultiInputStream();
+    const aecWarmupAudioFilter = new TransformStream({
+      transform: (frame, controller) => {
+        const shouldDiscardForAecWarmup = this.agentSession.agentState === "speaking" && this.agentSession._aecWarmupRemaining > 0;
+        if (!shouldDiscardForAecWarmup) {
+          controller.enqueue(frame);
+        }
+      }
+    });
     this.audioStreamId = this.audioStream.addInputStream(audioStream);
-    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.tee();
+    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.pipeThrough(aecWarmupAudioFilter).tee();
     if (this.realtimeSession) {
       this.realtimeSession.setInputAudioStream(realtimeAudioStream);
     }
@@ -435,6 +501,13 @@ class AgentActivity {
     this.logger.info("onInputSpeechStarted");
     if (!this.vad) {
       this.agentSession._updateUserState("speaking");
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfOverlapSpeech(
+          0,
+          Date.now(),
+          this.agentSession._userSpeakingSpan
+        );
+      }
     }
     try {
       this.interrupt();
@@ -448,6 +521,9 @@ class AgentActivity {
   onInputSpeechStopped(ev) {
     this.logger.info(ev, "onInputSpeechStopped");
     if (!this.vad) {
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onEndOfOverlapSpeech(Date.now(), this.agentSession._userSpeakingSpan);
+      }
       this.agentSession._updateUserState("listening");
     }
     if (ev.userTranscriptionEnabled) {
@@ -509,48 +585,75 @@ class AgentActivity {
   onStartOfSpeech(ev) {
     let speechStartTime = Date.now();
     if (ev) {
-      speechStartTime = speechStartTime - ev.speechDuration;
+      speechStartTime = speechStartTime - ev.speechDuration - ev.inferenceDuration;
     }
     this.agentSession._updateUserState("speaking", speechStartTime);
+    if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+      this.audioRecognition.onStartOfOverlapSpeech(
+        ev.speechDuration,
+        speechStartTime,
+        this.agentSession._userSpeakingSpan
+      );
+    }
   }
   onEndOfSpeech(ev) {
     let speechEndTime = Date.now();
     if (ev) {
-      speechEndTime = speechEndTime - ev.silenceDuration;
+      speechEndTime = speechEndTime - ev.silenceDuration - ev.inferenceDuration;
+    }
+    if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+      this.audioRecognition.onEndOfOverlapSpeech(
+        speechEndTime,
+        this.agentSession._userSpeakingSpan
+      );
     }
     this.agentSession._updateUserState("listening", speechEndTime);
   }
   onVADInferenceDone(ev) {
+    var _a;
     if (this.turnDetection === "manual" || this.turnDetection === "realtime_llm") {
       return;
     }
-    if (ev.speechDuration >= this.agentSession.options.minInterruptionDuration) {
+    if (ev.speechDuration >= ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.minDuration)) {
       this.interruptByAudioActivity();
     }
   }
   interruptByAudioActivity() {
-    var _a, _b;
+    var _a, _b, _c, _d;
+    if (!this.isInterruptionByAudioActivityEnabled) {
+      return;
+    }
+    if (this.agentSession._aecWarmupRemaining > 0) {
+      return;
+    }
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
       return;
     }
-    if (this.stt && this.agentSession.options.minInterruptionWords > 0 && this.audioRecognition) {
+    if (this.stt && ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0 && this.audioRecognition) {
       const text = this.audioRecognition.currentTranscript;
       const normalizedText = text ?? "";
       const wordCount = splitWords(normalizedText, true).length;
-      if (wordCount < this.agentSession.options.minInterruptionWords) {
+      if (wordCount < ((_b = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
         return;
       }
     }
-    (_a = this.realtimeSession) == null ? void 0 : _a.startUserActivity();
+    (_c = this.realtimeSession) == null ? void 0 : _c.startUserActivity();
     if (this._currentSpeech && !this._currentSpeech.interrupted && this._currentSpeech.allowInterruptions) {
       this.logger.info(
         { "speech id": this._currentSpeech.id },
         "speech interrupted by audio activity"
       );
-      (_b = this.realtimeSession) == null ? void 0 : _b.interrupt();
+      (_d = this.realtimeSession) == null ? void 0 : _d.interrupt();
       this._currentSpeech.interrupt();
     }
   }
+  onInterruption(ev) {
+    this.restoreInterruptionByAudioActivity();
+    this.interruptByAudioActivity();
+    if (this.audioRecognition) {
+      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.timestamp);
+    }
+  }
   onInterimTranscript(ev) {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
       return;
@@ -599,7 +702,8 @@ class AgentActivity {
     );
     const userMessage = ChatMessage.create({
       role: "user",
-      content: info.newTranscript
+      content: info.newTranscript,
+      transcriptConfidence: info.transcriptConfidence
     });
     const chatCtx = this.agent.chatCtx.copy();
     const speechHandle = this.generateReply({
@@ -657,6 +761,7 @@ class AgentActivity {
     return task;
   }
   async onEndOfTurn(info) {
+    var _a, _b;
     if (this.schedulingPaused) {
       this.cancelPreemptiveGeneration();
       this.logger.warn(
@@ -665,14 +770,14 @@ class AgentActivity {
       );
       return true;
     }
-    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && this.agentSession.options.minInterruptionWords > 0) {
+    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0) {
       const wordCount = splitWords(info.newTranscript, true).length;
-      if (wordCount < this.agentSession.options.minInterruptionWords) {
+      if (wordCount < ((_b = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
         this.cancelPreemptiveGeneration();
         this.logger.info(
           {
             wordCount,
-            minInterruptionWords: this.agentSession.options.minInterruptionWords
+            minInterruptionWords: this.agentSession.options.turnHandling.interruption.minWords
           },
           "skipping user input, word count below minimum interruption threshold"
         );
@@ -808,11 +913,18 @@ class AgentActivity {
         instructions = `${this.agent.instructions}
 ${instructions}`;
       }
+      const onEnterData = onEnterStorage.getStore();
+      const shouldFilterTools = (onEnterData == null ? void 0 : onEnterData.agent) === this.agent && (onEnterData == null ? void 0 : onEnterData.session) === this.agentSession;
+      const tools = shouldFilterTools ? Object.fromEntries(
+        Object.entries(this.agent.toolCtx).filter(
+          ([, fnTool]) => !(fnTool.flags & ToolFlag.IGNORE_ON_ENTER)
+        )
+      ) : this.agent.toolCtx;
       const task = this.createSpeechTask({
         taskFn: (abortController) => this.pipelineReplyTask(
           handle,
           chatCtx ?? this.agent.chatCtx,
-          this.agent.toolCtx,
+          tools,
           {
             toolChoice: toOaiToolChoice(toolChoice !== void 0 ? toolChoice : this.toolChoice)
           },
@@ -884,7 +996,8 @@ ${instructions}`;
     }
     let userMessage = ChatMessage.create({
       role: "user",
-      content: info.newTranscript
+      content: info.newTranscript,
+      transcriptConfidence: info.transcriptConfidence
     });
     const chatCtx = this.agent.chatCtx.copy();
     const startTime = Date.now();
@@ -902,11 +1015,32 @@ ${instructions}`;
     } else if (this.llm === void 0) {
       return;
     }
+    const userMetricsReport = {};
+    if (info.startedSpeakingAt !== void 0) {
+      userMetricsReport.startedSpeakingAt = info.startedSpeakingAt / 1e3;
+    }
+    if (info.stoppedSpeakingAt !== void 0) {
+      userMetricsReport.stoppedSpeakingAt = info.stoppedSpeakingAt / 1e3;
+    }
+    if (info.transcriptionDelay !== void 0) {
+      userMetricsReport.transcriptionDelay = info.transcriptionDelay / 1e3;
+    }
+    if (info.endOfUtteranceDelay !== void 0) {
+      userMetricsReport.endOfTurnDelay = info.endOfUtteranceDelay / 1e3;
+    }
+    userMetricsReport.onUserTurnCompletedDelay = callbackDuration / 1e3;
+    if (userMessage) {
+      userMessage.metrics = userMetricsReport;
+    }
     let speechHandle;
     if (this._preemptiveGeneration !== void 0) {
       const preemptive = this._preemptiveGeneration;
       if (preemptive.info.newTranscript === (userMessage == null ? void 0 : userMessage.textContent) && preemptive.chatCtx.isEquivalent(chatCtx) && isSameToolContext(preemptive.tools, this.tools) && isSameToolChoice(preemptive.toolChoice, this.toolChoice)) {
         speechHandle = preemptive.speechHandle;
+        if (preemptive.userMessage && userMessage) {
+          preemptive.userMessage.metrics = userMetricsReport;
+          preemptive.userMessage.transcriptConfidence = userMessage.transcriptConfidence;
+        }
         this.scheduleSpeech(speechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
         this.logger.debug(
           {
@@ -940,6 +1074,7 @@ ${instructions}`;
     );
   }
   async ttsTask(speechHandle, text, addToChatCtx, modelSettings, replyAbortController, audio) {
+    var _a, _b;
     speechHandle._agentTurnContext = otelContext.active();
     speechHandleStorage.enterWith(speechHandle);
     const transcriptionOutput = this.agentSession.output.transcriptionEnabled ? this.agentSession.output.transcription : null;
@@ -972,11 +1107,18 @@ ${instructions}`;
       textOut = _textOut;
       tasks.push(textForwardTask);
     }
+    let replyStartedSpeakingAt;
+    let replyTtsGenData = null;
     const onFirstFrame = (startedSpeakingAt) => {
+      replyStartedSpeakingAt = startedSpeakingAt ?? Date.now();
       this.agentSession._updateAgentState("speaking", {
         startTime: startedSpeakingAt,
         otelContext: speechHandle._agentTurnContext
       });
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfAgentSpeech();
+        this.isInterruptionByAudioActivityEnabled = false;
+      }
     };
     if (!audioOutput) {
       if (textOut) {
@@ -989,9 +1131,12 @@ ${instructions}`;
           (...args) => this.agent.ttsNode(...args),
           audioSource,
           modelSettings,
-          replyAbortController
+          replyAbortController,
+          (_a = this.tts) == null ? void 0 : _a.model,
+          (_b = this.tts) == null ? void 0 : _b.provider
         );
         tasks.push(ttsTask);
+        replyTtsGenData = ttsGenData;
         const [forwardTask, _audioOut] = performAudioForwarding(
           ttsGenData.audioStream,
           audioOutput,
@@ -1023,16 +1168,30 @@ ${instructions}`;
       }
     }
     if (addToChatCtx) {
+      const replyStoppedSpeakingAt = Date.now();
+      const replyAssistantMetrics = {};
+      if ((replyTtsGenData == null ? void 0 : replyTtsGenData.ttfb) !== void 0) {
+        replyAssistantMetrics.ttsNodeTtfb = replyTtsGenData.ttfb;
+      }
+      if (replyStartedSpeakingAt !== void 0) {
+        replyAssistantMetrics.startedSpeakingAt = replyStartedSpeakingAt / 1e3;
+        replyAssistantMetrics.stoppedSpeakingAt = replyStoppedSpeakingAt / 1e3;
+      }
       const message = ChatMessage.create({
         role: "assistant",
         content: (textOut == null ? void 0 : textOut.text) || "",
-        interrupted: speechHandle.interrupted
+        interrupted: speechHandle.interrupted,
+        metrics: replyAssistantMetrics
       });
       this.agent._chatCtx.insert(message);
       this.agentSession._conversationItemAdded(message);
     }
     if (this.agentSession.agentState === "speaking") {
       this.agentSession._updateAgentState("listening");
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onEndOfAgentSpeech(Date.now());
+      }
+      this.restoreInterruptionByAudioActivity();
     }
   }
   _pipelineReplyTaskImpl = async ({
@@ -1044,9 +1203,10 @@ ${instructions}`;
     instructions,
     newMessage,
     toolsMessages,
-    span
+    span,
+    _previousUserMetrics
   }) => {
-    var _a, _b;
+    var _a, _b, _c, _d, _e, _f;
     speechHandle._agentTurnContext = otelContext.active();
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     if (instructions) {
@@ -1084,7 +1244,9 @@ ${instructions}`;
       chatCtx,
       toolCtx,
       modelSettings,
-      replyAbortController
+      replyAbortController,
+      (_b = this.llm) == null ? void 0 : _b.model,
+      (_c = this.llm) == null ? void 0 : _c.provider
     );
     tasks.push(llmTask);
     let ttsTask = null;
@@ -1097,16 +1259,20 @@ ${instructions}`;
         (...args) => this.agent.ttsNode(...args),
         ttsTextInput,
         modelSettings,
-        replyAbortController
+        replyAbortController,
+        (_d = this.tts) == null ? void 0 : _d.model,
+        (_e = this.tts) == null ? void 0 : _e.provider
       );
       tasks.push(ttsTask);
     } else {
       llmOutput = llmGenData.textStream;
     }
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForScheduled()]);
+    let userMetrics = _previousUserMetrics;
     if (newMessage && speechHandle.scheduled) {
       this.agent._chatCtx.insert(newMessage);
       this.agentSession._conversationItemAdded(newMessage);
+      userMetrics = newMessage.metrics;
     }
     if (speechHandle.interrupted) {
       replyAbortController.abort();
@@ -1118,7 +1284,7 @@ ${instructions}`;
     speechHandle._clearAuthorization();
     const replyStartedAt = Date.now();
     let transcriptionInput = llmOutput;
-    if (this.useTtsAlignedTranscript && ((_b = this.tts) == null ? void 0 : _b.capabilities.alignedTranscript) && ttsGenData) {
+    if (this.useTtsAlignedTranscript && ((_f = this.tts) == null ? void 0 : _f.capabilities.alignedTranscript) && ttsGenData) {
       const timedTextsStream = await Promise.race([
         ttsGenData.timedTextsFut.await,
         (ttsTask == null ? void 0 : ttsTask.result.catch(
@@ -1141,11 +1307,17 @@ ${instructions}`;
       tasks.push(textForwardTask);
       textOut = _textOut;
     }
+    let agentStartedSpeakingAt;
     const onFirstFrame = (startedSpeakingAt) => {
+      agentStartedSpeakingAt = startedSpeakingAt ?? Date.now();
       this.agentSession._updateAgentState("speaking", {
         startTime: startedSpeakingAt,
         otelContext: speechHandle._agentTurnContext
       });
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfAgentSpeech();
+        this.isInterruptionByAudioActivityEnabled = false;
+      }
     };
     let audioOut = null;
     if (audioOutput) {
@@ -1188,6 +1360,25 @@ ${instructions}`;
     if (audioOutput) {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
     }
+    const agentStoppedSpeakingAt = Date.now();
+    const assistantMetrics = {};
+    if (llmGenData.ttft !== void 0) {
+      assistantMetrics.llmNodeTtft = llmGenData.ttft;
+    }
+    if ((ttsGenData == null ? void 0 : ttsGenData.ttfb) !== void 0) {
+      assistantMetrics.ttsNodeTtfb = ttsGenData.ttfb;
+    }
+    if (agentStartedSpeakingAt !== void 0) {
+      assistantMetrics.startedSpeakingAt = agentStartedSpeakingAt / 1e3;
+      assistantMetrics.stoppedSpeakingAt = agentStoppedSpeakingAt / 1e3;
+      if ((userMetrics == null ? void 0 : userMetrics.stoppedSpeakingAt) !== void 0) {
+        const e2eLatency = agentStartedSpeakingAt / 1e3 - userMetrics.stoppedSpeakingAt;
+        assistantMetrics.e2eLatency = e2eLatency;
+        span.setAttribute(traceTypes.ATTR_E2E_LATENCY, e2eLatency);
+      }
+    }
+    span.setAttribute(traceTypes.ATTR_SPEECH_INTERRUPTED, speechHandle.interrupted);
+    let hasSpeechMessage = false;
     if (toolsMessages) {
       for (const msg of toolsMessages) {
         msg.createdAt = replyStartedAt;
@@ -1228,20 +1419,27 @@ ${instructions}`;
         }
       }
       if (forwardedText) {
+        hasSpeechMessage = true;
         const message = ChatMessage.create({
           role: "assistant",
           content: forwardedText,
           id: llmGenData.id,
           interrupted: true,
-          createdAt: replyStartedAt
+          createdAt: replyStartedAt,
+          metrics: assistantMetrics
         });
         chatCtx.insert(message);
         this.agent._chatCtx.insert(message);
         speechHandle._itemAdded([message]);
         this.agentSession._conversationItemAdded(message);
+        span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, forwardedText);
       }
       if (this.agentSession.agentState === "speaking") {
         this.agentSession._updateAgentState("listening");
+        if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+          this.restoreInterruptionByAudioActivity();
+        }
       }
       this.logger.info(
         { speech_id: speechHandle.id, message: forwardedText },
@@ -1252,17 +1450,20 @@ ${instructions}`;
       return;
     }
     if (textOut && textOut.text) {
+      hasSpeechMessage = true;
       const message = ChatMessage.create({
         role: "assistant",
         id: llmGenData.id,
         interrupted: false,
         createdAt: replyStartedAt,
-        content: textOut.text
+        content: textOut.text,
+        metrics: assistantMetrics
       });
       chatCtx.insert(message);
       this.agent._chatCtx.insert(message);
       speechHandle._itemAdded([message]);
       this.agentSession._conversationItemAdded(message);
+      span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, textOut.text);
       this.logger.info(
         { speech_id: speechHandle.id, message: textOut.text },
         "playout completed without interruption"
@@ -1272,6 +1473,12 @@ ${instructions}`;
       this.agentSession._updateAgentState("thinking");
     } else if (this.agentSession.agentState === "speaking") {
       this.agentSession._updateAgentState("listening");
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+          this.restoreInterruptionByAudioActivity();
+        }
+      }
     }
     speechHandle._markGenerationDone();
     await executeToolsTask.result;
@@ -1311,7 +1518,8 @@ ${instructions}`;
           replyAbortController,
           instructions,
           void 0,
-          toolMessages
+          toolMessages,
+          hasSpeechMessage ? void 0 : userMetrics
         ),
         ownedSpeechHandle: speechHandle,
         name: "AgentActivity.pipelineReply"
@@ -1331,7 +1539,7 @@ ${instructions}`;
       }
     }
   };
-  pipelineReplyTask = async (speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages) => tracer.startActiveSpan(
+  pipelineReplyTask = async (speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages, _previousUserMetrics) => tracer.startActiveSpan(
     async (span) => this._pipelineReplyTaskImpl({
       speechHandle,
       chatCtx,
@@ -1341,7 +1549,8 @@ ${instructions}`;
       instructions,
       newMessage,
       toolsMessages,
-      span
+      span,
+      _previousUserMetrics
     }),
     {
       name: "agent_turn",
@@ -1407,6 +1616,7 @@ ${instructions}`;
       });
     };
     const readMessages = async (abortController, outputs) => {
+      var _a2, _b;
       replyAbortController.signal.addEventListener("abort", () => abortController.abort(), {
         once: true
       });
@@ -1453,7 +1663,9 @@ ${instructions}`;
                 (...args) => this.agent.ttsNode(...args),
                 ttsTextInput,
                 modelSettings,
-                abortController
+                abortController,
+                (_a2 = this.tts) == null ? void 0 : _a2.model,
+                (_b = this.tts) == null ? void 0 : _b.provider
               );
               tasks.push(ttsTask);
               realtimeAudioResult = ttsGenData.audioStream;
@@ -1845,11 +2057,46 @@ ${instructions}`;
       if (this._mainTask) {
         await this._mainTask.cancelAndWait();
       }
+      if (this.interruptionDetector) {
+        this.interruptionDetector.off(
+          "user_overlapping_speech",
+          this.onInterruptionOverlappingSpeech
+        );
+        this.interruptionDetector.off("metrics_collected", this.onInterruptionMetricsCollected);
+        this.interruptionDetector.off("error", this.onInterruptionError);
+      }
       this.agent._agentActivity = void 0;
     } finally {
       unlock();
     }
   }
+  resolveInterruptionDetector() {
+    const interruptionDetection = this.agent.interruptionDetection ?? this.agentSession.interruptionDetection;
+    if (!(this.stt && this.stt.capabilities.alignedTranscript && this.stt.capabilities.streaming && this.vad && this.turnDetection !== "manual" && this.turnDetection !== "realtime_llm" && !(this.llm instanceof RealtimeModel))) {
+      if (interruptionDetection === "adaptive") {
+        this.logger.warn(
+          "interruptionDetection is provided, but it's not compatible with the current configuration and will be disabled"
+        );
+        return void 0;
+      }
+    }
+    if (interruptionDetection !== void 0 && interruptionDetection === false || interruptionDetection === "vad") {
+      return void 0;
+    }
+    try {
+      const detector = new AdaptiveInterruptionDetector();
+      detector.on("user_overlapping_speech", this.onInterruptionOverlappingSpeech);
+      detector.on("metrics_collected", this.onInterruptionMetricsCollected);
+      detector.on("error", this.onInterruptionError);
+      return detector;
+    } catch (error) {
+      this.logger.warn({ error }, "could not instantiate AdaptiveInterruptionDetector");
+    }
+    return void 0;
+  }
+  restoreInterruptionByAudioActivity() {
+    this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
+  }
   async _closeSessionResources() {
     var _a, _b, _c;
     if (this.llm instanceof LLM) {
@@ -1891,6 +2138,7 @@ function toOaiToolChoice(toolChoice) {
 }
 export {
   AgentActivity,
-  agentActivityStorage
+  agentActivityStorage,
+  onEnterStorage
 };
 //# sourceMappingURL=agent_activity.js.map