@livekit/agents 1.1.0 → 1.2.1

package/dist/voice/agent_activity.js

@@ -1,20 +1,31 @@
 import { Mutex } from "@livekit/mutex";
+import { ROOT_CONTEXT, context as otelContext, trace } from "@opentelemetry/api";
 import { Heap } from "heap-js";
 import { AsyncLocalStorage } from "node:async_hooks";
-import { ReadableStream } from "node:stream/web";
+import { ReadableStream, TransformStream } from "node:stream/web";
+import { AdaptiveInterruptionDetector } from "../inference/interruption/interruption_detector.js";
 import { ChatMessage } from "../llm/chat_context.js";
 import {
   LLM,
-  RealtimeModel
+  RealtimeModel,
+  ToolFlag
 } from "../llm/index.js";
+import { isSameToolChoice, isSameToolContext } from "../llm/tool_context.js";
 import { log } from "../log.js";
-import { DeferredReadableStream } from "../stream/deferred_stream.js";
+import { MultiInputStream } from "../stream/multi_input_stream.js";
 import { STT } from "../stt/stt.js";
+import { recordRealtimeMetrics, traceTypes, tracer } from "../telemetry/index.js";
 import { splitWords } from "../tokenize/basic/word.js";
 import { TTS } from "../tts/tts.js";
-import { Future, Task, cancelAndWait, waitFor } from "../utils.js";
+import { Future, Task, cancelAndWait, isDevMode, isHosted, waitFor } from "../utils.js";
 import { VAD } from "../vad.js";
-import { StopResponse, asyncLocalStorage } from "./agent.js";
+import {
+  StopResponse,
+  _getActivityTaskInfo,
+  _setActivityTaskInfo,
+  functionCallStorage,
+  speechHandleStorage
+} from "./agent.js";
 import {} from "./agent_session.js";
 import {
   AudioRecognition
@@ -37,28 +48,66 @@ import {
   updateInstructions
 } from "./generation.js";
 import { SpeechHandle } from "./speech_handle.js";
-const speechHandleStorage = new AsyncLocalStorage();
+import { setParticipantSpanAttributes } from "./utils.js";
+const agentActivityStorage = new AsyncLocalStorage();
+const onEnterStorage = new AsyncLocalStorage();
 class AgentActivity {
+  agent;
+  agentSession;
   static REPLY_TASK_CANCEL_TIMEOUT = 5e3;
   started = false;
   audioRecognition;
   realtimeSession;
+  realtimeSpans;
+  // Maps response_id to OTEL span for metrics recording
   turnDetectionMode;
   logger = log();
-  _draining = false;
+  _schedulingPaused = true;
+  _drainBlockedTasks = [];
   _currentSpeech;
   speechQueue;
   // [priority, timestamp, speechHandle]
   q_updated;
   speechTasks = /* @__PURE__ */ new Set();
   lock = new Mutex();
-  audioStream = new DeferredReadableStream();
+  audioStream = new MultiInputStream();
+  audioStreamId;
   // default to null as None, which maps to the default provider tool choice value
   toolChoice = null;
-  agent;
-  agentSession;
+  _preemptiveGeneration;
+  interruptionDetector;
+  isInterruptionDetectionEnabled;
+  isInterruptionByAudioActivityEnabled;
+  isDefaultInterruptionByAudioActivityEnabled;
+  onRealtimeGenerationCreated = (ev) => this.onGenerationCreated(ev);
+  onRealtimeInputSpeechStarted = (ev) => this.onInputSpeechStarted(ev);
+  onRealtimeInputSpeechStopped = (ev) => this.onInputSpeechStopped(ev);
+  onRealtimeInputAudioTranscriptionCompleted = (ev) => this.onInputAudioTranscriptionCompleted(ev);
+  onModelError = (ev) => this.onError(ev);
+  onInterruptionOverlappingSpeech = (ev) => {
+    this.agentSession.emit(AgentSessionEventTypes.OverlappingSpeech, ev);
+  };
+  onInterruptionMetricsCollected = (ev) => {
+    this.agentSession._usageCollector.collect(ev);
+    this.agentSession.emit(
+      AgentSessionEventTypes.MetricsCollected,
+      createMetricsCollectedEvent({ metrics: ev })
+    );
+  };
+  onInterruptionError = (ev) => {
+    const errorEvent = createErrorEvent(ev, this.interruptionDetector);
+    this.agentSession.emit(AgentSessionEventTypes.Error, errorEvent);
+    if (!ev.recoverable) {
+      this.agentSession._onError(ev);
+      this.fallbackToVadInterruption();
+      return;
+    }
+    this.agentSession._onError(ev);
+  };
   /** @internal */
   _mainTask;
+  _onEnterTask;
+  _onExitTask;
   _userTurnCompletedTask;
   constructor(agent, agentSession) {
     this.agent = agent;
@@ -70,7 +119,7 @@ class AgentActivity {
     this.turnDetectionMode = typeof this.turnDetection === "string" ? this.turnDetection : void 0;
     if (this.turnDetectionMode === "vad" && this.vad === void 0) {
       this.logger.warn(
-        'turnDetection is set to "vad", but no VAD model is provided, ignoring the turnDdetection setting'
+        'turnDetection is set to "vad", but no VAD model is provided, ignoring the turnDetection setting'
       );
       this.turnDetectionMode = void 0;
     }
@@ -113,89 +162,136 @@ class AgentActivity {
       );
       this.turnDetectionMode = void 0;
     }
-    if (!this.vad && this.stt && this.llm instanceof LLM && this.allowInterruptions && this.turnDetectionMode === void 0) {
+    if (!this.vad && this.stt && !this.stt.capabilities.streaming && this.llm instanceof LLM && this.allowInterruptions && this.turnDetectionMode === void 0) {
       this.logger.warn(
-        "VAD is not set. Enabling VAD is recommended when using LLM and STT for more responsive interruption handling."
+        "VAD is not set. Enabling VAD is recommended when using LLM and non-streaming STT for more responsive interruption handling."
       );
     }
+    this.interruptionDetector = this.resolveInterruptionDetector();
+    this.isInterruptionDetectionEnabled = !!this.interruptionDetector;
+    this.isInterruptionByAudioActivityEnabled = this.turnDetectionMode !== "manual" && this.turnDetectionMode !== "realtime_llm";
+    this.isDefaultInterruptionByAudioActivityEnabled = this.isInterruptionByAudioActivityEnabled;
   }
   async start() {
     const unlock = await this.lock.lock();
     try {
-      this.agent._agentActivity = this;
-      if (this.llm instanceof RealtimeModel) {
-        this.realtimeSession = this.llm.session();
-        this.realtimeSession.on("generation_created", (ev) => this.onGenerationCreated(ev));
-        this.realtimeSession.on("input_speech_started", (ev) => this.onInputSpeechStarted(ev));
-        this.realtimeSession.on("input_speech_stopped", (ev) => this.onInputSpeechStopped(ev));
-        this.realtimeSession.on(
-          "input_audio_transcription_completed",
-          (ev) => this.onInputAudioTranscriptionCompleted(ev)
+      await this._startSession({ spanName: "start_agent_activity", runOnEnter: true });
+    } finally {
+      unlock();
+    }
+  }
+  async resume() {
+    const unlock = await this.lock.lock();
+    try {
+      await this._startSession({ spanName: "resume_agent_activity", runOnEnter: false });
+    } finally {
+      unlock();
+    }
+  }
+  async _startSession(options) {
+    var _a, _b, _c, _d, _e;
+    const { spanName, runOnEnter } = options;
+    const startSpan = tracer.startSpan({
+      name: spanName,
+      attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+      context: ROOT_CONTEXT
+    });
+    this.agent._agentActivity = this;
+    if (this.llm instanceof RealtimeModel) {
+      this.realtimeSession = this.llm.session();
+      this.realtimeSpans = /* @__PURE__ */ new Map();
+      this.realtimeSession.on("generation_created", this.onRealtimeGenerationCreated);
+      this.realtimeSession.on("input_speech_started", this.onRealtimeInputSpeechStarted);
+      this.realtimeSession.on("input_speech_stopped", this.onRealtimeInputSpeechStopped);
+      this.realtimeSession.on(
+        "input_audio_transcription_completed",
+        this.onRealtimeInputAudioTranscriptionCompleted
+      );
+      this.realtimeSession.on("metrics_collected", this.onMetricsCollected);
+      this.realtimeSession.on("error", this.onModelError);
+      removeInstructions(this.agent._chatCtx);
+      try {
+        await this.realtimeSession.updateInstructions(this.agent.instructions);
+      } catch (error) {
+        this.logger.error(error, "failed to update the instructions");
+      }
+      try {
+        await this.realtimeSession.updateChatCtx(this.agent.chatCtx);
+      } catch (error) {
+        this.logger.error(error, "failed to update the chat context");
+      }
+      try {
+        await this.realtimeSession.updateTools(this.tools);
+      } catch (error) {
+        this.logger.error(error, "failed to update the tools");
+      }
+      if (!this.llm.capabilities.audioOutput && !this.tts && this.agentSession.output.audio) {
+        this.logger.error(
+          "audio output is enabled but RealtimeModel has no audio modality and no TTS is set. Either enable audio modality in the RealtimeModel or set a TTS model."
         );
-        this.realtimeSession.on("metrics_collected", (ev) => this.onMetricsCollected(ev));
-        this.realtimeSession.on("error", (ev) => this.onError(ev));
-        removeInstructions(this.agent._chatCtx);
-        try {
-          await this.realtimeSession.updateInstructions(this.agent.instructions);
-        } catch (error) {
-          this.logger.error(error, "failed to update the instructions");
-        }
-        try {
-          await this.realtimeSession.updateChatCtx(this.agent.chatCtx);
-        } catch (error) {
-          this.logger.error(error, "failed to update the chat context");
-        }
-        try {
-          await this.realtimeSession.updateTools(this.tools);
-        } catch (error) {
-          this.logger.error(error, "failed to update the tools");
-        }
-      } else if (this.llm instanceof LLM) {
-        try {
-          updateInstructions({
-            chatCtx: this.agent._chatCtx,
-            instructions: this.agent.instructions,
-            addIfMissing: true
-          });
-        } catch (error) {
-          this.logger.error("failed to update the instructions", error);
-        }
       }
-      if (this.llm instanceof LLM) {
-        this.llm.on("metrics_collected", (ev) => this.onMetricsCollected(ev));
-        this.llm.on("error", (ev) => this.onError(ev));
-      }
-      if (this.stt instanceof STT) {
-        this.stt.on("metrics_collected", (ev) => this.onMetricsCollected(ev));
-        this.stt.on("error", (ev) => this.onError(ev));
-      }
-      if (this.tts instanceof TTS) {
-        this.tts.on("metrics_collected", (ev) => this.onMetricsCollected(ev));
-        this.tts.on("error", (ev) => this.onError(ev));
-      }
-      if (this.vad instanceof VAD) {
-        this.vad.on("metrics_collected", (ev) => this.onMetricsCollected(ev));
-      }
-      this.audioRecognition = new AudioRecognition({
-        recognitionHooks: this,
-        // Disable stt node if stt is not provided
-        stt: this.stt ? (...args) => this.agent.sttNode(...args) : void 0,
-        vad: this.vad,
-        turnDetector: typeof this.turnDetection === "string" ? void 0 : this.turnDetection,
-        turnDetectionMode: this.turnDetectionMode,
-        minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
-        maxEndpointingDelay: this.agentSession.options.maxEndpointingDelay
-      });
-      this.audioRecognition.start();
-      this.started = true;
-      this._mainTask = Task.from(({ signal }) => this.mainTask(signal));
-      this.createSpeechTask({
-        task: Task.from(() => this.agent.onEnter()),
+    } else if (this.llm instanceof LLM) {
+      try {
+        updateInstructions({
+          chatCtx: this.agent._chatCtx,
+          instructions: this.agent.instructions,
+          addIfMissing: true
+        });
+      } catch (error) {
+        this.logger.error("failed to update the instructions", error);
+      }
+    }
+    if (this.llm instanceof LLM) {
+      this.llm.on("metrics_collected", this.onMetricsCollected);
+      this.llm.on("error", this.onModelError);
+    }
+    if (this.stt instanceof STT) {
+      this.stt.on("metrics_collected", this.onMetricsCollected);
+      this.stt.on("error", this.onModelError);
+    }
+    if (this.tts instanceof TTS) {
+      this.tts.on("metrics_collected", this.onMetricsCollected);
+      this.tts.on("error", this.onModelError);
+    }
+    if (this.vad instanceof VAD) {
+      this.vad.on("metrics_collected", this.onMetricsCollected);
+    }
+    this.audioRecognition = new AudioRecognition({
+      recognitionHooks: this,
+      // Disable stt node if stt is not provided
+      stt: this.stt ? (...args) => this.agent.sttNode(...args) : void 0,
+      vad: this.vad,
+      turnDetector: typeof this.turnDetection === "string" ? void 0 : this.turnDetection,
+      turnDetectionMode: this.turnDetectionMode,
+      interruptionDetection: this.interruptionDetector,
+      minEndpointingDelay: ((_b = (_a = this.agent.turnHandling) == null ? void 0 : _a.endpointing) == null ? void 0 : _b.minDelay) ?? this.agentSession.sessionOptions.turnHandling.endpointing.minDelay,
+      maxEndpointingDelay: ((_d = (_c = this.agent.turnHandling) == null ? void 0 : _c.endpointing) == null ? void 0 : _d.maxDelay) ?? this.agentSession.sessionOptions.turnHandling.endpointing.maxDelay,
+      rootSpanContext: this.agentSession.rootSpanContext,
+      sttModel: (_e = this.stt) == null ? void 0 : _e.label,
+      sttProvider: this.getSttProvider(),
+      getLinkedParticipant: () => {
+        var _a2;
+        return (_a2 = this.agentSession._roomIO) == null ? void 0 : _a2.linkedParticipant;
+      }
+    });
+    this.audioRecognition.start();
+    this.started = true;
+    this._resumeSchedulingTask();
+    if (runOnEnter) {
+      this._onEnterTask = this.createSpeechTask({
+        taskFn: () => onEnterStorage.run(
+          { session: this.agentSession, agent: this.agent },
+          () => tracer.startActiveSpan(async () => this.agent.onEnter(), {
+            name: "on_enter",
+            context: trace.setSpan(ROOT_CONTEXT, startSpan),
+            attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id }
+          })
+        ),
+        inlineTask: true,
         name: "AgentActivity_onEnter"
       });
-    } finally {
-      unlock();
     }
+    startSpan.end();
   }
   get currentSpeech() {
     return this._currentSpeech;
@@ -206,6 +302,15 @@ class AgentActivity {
   get stt() {
     return this.agent.stt || this.agentSession.stt;
   }
+  getSttProvider() {
+    var _a;
+    const label = (_a = this.stt) == null ? void 0 : _a.label;
+    if (!label) {
+      return void 0;
+    }
+    const [provider] = label.split("-", 1);
+    return provider || label;
+  }
   get llm() {
     return this.agent.llm || this.agentSession.llm;
   }
@@ -215,21 +320,46 @@ class AgentActivity {
   get tools() {
     return this.agent.toolCtx;
   }
-  get draining() {
-    return this._draining;
+  get schedulingPaused() {
+    return this._schedulingPaused;
   }
   get realtimeLLMSession() {
     return this.realtimeSession;
   }
   get allowInterruptions() {
-    return this.agentSession.options.allowInterruptions;
+    var _a, _b;
+    return ((_b = (_a = this.agent.turnHandling) == null ? void 0 : _a.interruption) == null ? void 0 : _b.enabled) ?? this.agentSession.sessionOptions.turnHandling.interruption.enabled;
+  }
+  get useTtsAlignedTranscript() {
+    return this.agent.useTtsAlignedTranscript ?? this.agentSession.useTtsAlignedTranscript;
   }
   get turnDetection() {
-    return this.agentSession.turnDetection;
+    var _a;
+    return ((_a = this.agent.turnHandling) == null ? void 0 : _a.turnDetection) ?? this.agentSession.turnDetection;
+  }
+  get turnHandling() {
+    return this.agent.turnHandling ?? this.agentSession.sessionOptions.turnHandling;
   }
+  // get minEndpointingDelay(): number {
+  //   return (
+  //     this.agent.turnHandling?.endpointing?.minDelay ??
+  //     this.agentSession.sessionOptions.turnHandling.endpointing.minDelay
+  //   );
+  // }
+  // get maxEndpointingDelay(): number {
+  //   return (
+  //     this.agent.turnHandling?.endpointing?.maxDelay ??
+  //     this.agentSession.sessionOptions.turnHandling.endpointing.maxDelay
+  //   );
+  // }
   get toolCtx() {
     return this.agent.toolCtx;
   }
+  /** @internal */
+  get inputStartedAt() {
+    var _a;
+    return (_a = this.audioRecognition) == null ? void 0 : _a.inputStartedAt;
+  }
   async updateChatCtx(chatCtx) {
     chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });
     this.agent._chatCtx = chatCtx;
@@ -244,36 +374,79 @@ class AgentActivity {
       });
     }
   }
-  updateOptions({ toolChoice }) {
+  // TODO: Add when AgentConfigUpdate is ported to ChatContext.
+  async updateTools(tools) {
+    this.agent._tools = { ...tools };
+    if (this.realtimeSession) {
+      await this.realtimeSession.updateTools(tools);
+    }
+    if (this.llm instanceof LLM) {
+      await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: tools }));
+    }
+  }
+  updateOptions({
+    toolChoice,
+    turnDetection
+  }) {
     if (toolChoice !== void 0) {
       this.toolChoice = toolChoice;
     }
     if (this.realtimeSession) {
       this.realtimeSession.updateOptions({ toolChoice: this.toolChoice });
     }
+    if (turnDetection !== void 0) {
+      this.turnDetectionMode = turnDetection;
+      this.isDefaultInterruptionByAudioActivityEnabled = this.turnDetectionMode !== "manual" && this.turnDetectionMode !== "realtime_llm";
+      if (this.agentSession.agentState !== "speaking") {
+        this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
+      }
+    }
+    if (this.audioRecognition) {
+      this.audioRecognition.updateOptions({ turnDetection: this.turnDetectionMode });
+    }
   }
   attachAudioInput(audioStream) {
-    if (this.audioStream.isSourceSet) {
-      this.logger.debug("detaching existing audio input in agent activity");
-      this.audioStream.detachSource();
-    }
-    this.audioStream.setSource(audioStream);
-    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.tee();
-    if (this.realtimeSession) {
+    void this.audioStream.close();
+    this.audioStream = new MultiInputStream();
+    const aecWarmupAudioFilter = new TransformStream({
+      transform: (frame, controller) => {
+        const shouldDiscardForAecWarmup = this.agentSession.agentState === "speaking" && this.agentSession._aecWarmupRemaining > 0;
+        if (!shouldDiscardForAecWarmup) {
+          controller.enqueue(frame);
+        }
+      }
+    });
+    this.audioStreamId = this.audioStream.addInputStream(audioStream);
+    if (this.realtimeSession && this.audioRecognition) {
+      const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.pipeThrough(aecWarmupAudioFilter).tee();
       this.realtimeSession.setInputAudioStream(realtimeAudioStream);
-    }
-    if (this.audioRecognition) {
       this.audioRecognition.setInputAudioStream(recognitionAudioStream);
+    } else if (this.realtimeSession) {
+      this.realtimeSession.setInputAudioStream(
+        this.audioStream.stream.pipeThrough(aecWarmupAudioFilter)
+      );
+    } else if (this.audioRecognition) {
+      this.audioRecognition.setInputAudioStream(
+        this.audioStream.stream.pipeThrough(aecWarmupAudioFilter)
+      );
     }
   }
   detachAudioInput() {
-    this.audioStream.detachSource();
+    if (this.audioStreamId === void 0) {
+      return;
+    }
+    void this.audioStream.close();
+    this.audioStream = new MultiInputStream();
+    this.audioStreamId = void 0;
   }
-  commitUserTurn() {
+  commitUserTurn(options = {}) {
+    const { audioDetached = false, throwIfNotReady = true } = options;
     if (!this.audioRecognition) {
-      throw new Error("AudioRecognition is not initialized");
+      if (throwIfNotReady) {
+        throw new Error("AudioRecognition is not initialized");
+      }
+      return;
     }
-    const audioDetached = false;
     this.audioRecognition.commitUserTurn(audioDetached);
   }
   clearUserTurn() {
@@ -309,13 +482,11 @@ class AgentActivity {
       })
     );
     const task = this.createSpeechTask({
-      task: Task.from(
-        (abortController) => this.ttsTask(handle, text, addToChatCtx, {}, abortController, audio)
-      ),
+      taskFn: (abortController) => this.ttsTask(handle, text, addToChatCtx, {}, abortController, audio),
       ownedSpeechHandle: handle,
       name: "AgentActivity.say_tts"
     });
-    task.finally(() => this.onPipelineReplyDone());
+    task.result.finally(() => this.onPipelineReplyDone());
     this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
     return handle;
   }
@@ -325,6 +496,14 @@ class AgentActivity {
     if (speechHandle && (ev.type === "llm_metrics" || ev.type === "tts_metrics")) {
       ev.speechId = speechHandle.id;
     }
+    if (ev.type === "realtime_model_metrics" && this.realtimeSpans) {
+      const span = this.realtimeSpans.get(ev.requestId);
+      if (span) {
+        recordRealtimeMetrics(span, ev);
+        this.realtimeSpans.delete(ev.requestId);
+      }
+    }
+    this.agentSession._usageCollector.collect(ev);
     this.agentSession.emit(
       AgentSessionEventTypes.MetricsCollected,
       createMetricsCollectedEvent({ metrics: ev })
@@ -351,6 +530,13 @@ class AgentActivity {
     this.logger.info("onInputSpeechStarted");
     if (!this.vad) {
       this.agentSession._updateUserState("speaking");
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfOverlapSpeech(
+          0,
+          Date.now(),
+          this.agentSession._userSpeakingSpan
+        );
+      }
     }
     try {
       this.interrupt();
@@ -364,6 +550,9 @@ class AgentActivity {
   onInputSpeechStopped(ev) {
     this.logger.info(ev, "onInputSpeechStopped");
     if (!this.vad) {
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onEndOfOverlapSpeech(Date.now(), this.agentSession._userSpeakingSpan);
+      }
       this.agentSession._updateUserState("listening");
     }
     if (ev.userTranscriptionEnabled) {
@@ -398,8 +587,8 @@ class AgentActivity {
     if (ev.userInitiated) {
       return;
     }
-    if (this.draining) {
-      this.logger.warn("skipping new realtime generation, the agent is draining");
+    if (this.schedulingPaused) {
+      this.logger.warn("skipping new realtime generation, the speech scheduling is not running");
       return;
     }
     const handle = SpeechHandle.create({
@@ -415,45 +604,91 @@ class AgentActivity {
     );
     this.logger.info({ speech_id: handle.id }, "Creating speech handle");
     this.createSpeechTask({
-      task: Task.from(
-        (abortController) => this.realtimeGenerationTask(handle, ev, {}, abortController)
-      ),
+      taskFn: (abortController) => this.realtimeGenerationTask(handle, ev, {}, abortController),
       ownedSpeechHandle: handle,
       name: "AgentActivity.realtimeGeneration"
     });
     this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
   }
   // recognition hooks
-  onStartOfSpeech(_ev) {
-    this.agentSession._updateUserState("speaking");
+  onStartOfSpeech(ev) {
+    let speechStartTime = Date.now();
+    if (ev) {
+      speechStartTime = speechStartTime - ev.speechDuration - ev.inferenceDuration;
+    }
+    this.agentSession._updateUserState("speaking", {
+      lastSpeakingTime: speechStartTime,
+      otelContext: otelContext.active()
+    });
+    if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+      this.audioRecognition.onStartOfOverlapSpeech(
+        ev.speechDuration,
+        speechStartTime,
+        this.agentSession._userSpeakingSpan
+      );
+    }
   }
-  onEndOfSpeech(_ev) {
-    this.agentSession._updateUserState("listening");
+  onEndOfSpeech(ev) {
+    let speechEndTime = Date.now();
+    if (ev) {
+      speechEndTime = speechEndTime - ev.silenceDuration - ev.inferenceDuration;
+    }
+    if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+      this.audioRecognition.onEndOfOverlapSpeech(
+        speechEndTime,
+        this.agentSession._userSpeakingSpan
+      );
+    }
+    this.agentSession._updateUserState("listening", {
+      lastSpeakingTime: speechEndTime,
+      otelContext: otelContext.active()
+    });
   }
   onVADInferenceDone(ev) {
-    var _a, _b;
+    var _a;
     if (this.turnDetection === "manual" || this.turnDetection === "realtime_llm") {
       return;
     }
-    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
+    if (ev.speechDuration >= ((_a = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _a.minDuration)) {
+      this.interruptByAudioActivity();
+    }
+  }
+  interruptByAudioActivity() {
+    var _a, _b, _c, _d;
+    if (!this.isInterruptionByAudioActivityEnabled) {
+      return;
+    }
+    if (this.agentSession._aecWarmupRemaining > 0) {
       return;
     }
-    if (ev.speechDuration < this.agentSession.options.minInterruptionDuration) {
+    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
       return;
     }
-    if (this.stt && this.agentSession.options.minInterruptionWords > 0 && this.audioRecognition) {
+    if (this.stt && ((_a = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0 && this.audioRecognition) {
       const text = this.audioRecognition.currentTranscript;
-      if (text && splitWords(text, true).length < this.agentSession.options.minInterruptionWords) {
+      const normalizedText = text ?? "";
+      const wordCount = splitWords(normalizedText, true).length;
+      if (wordCount < ((_b = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
         return;
       }
     }
-    (_a = this.realtimeSession) == null ? void 0 : _a.startUserActivity();
+    (_c = this.realtimeSession) == null ? void 0 : _c.startUserActivity();
     if (this._currentSpeech && !this._currentSpeech.interrupted && this._currentSpeech.allowInterruptions) {
-      this.logger.info({ "speech id": this._currentSpeech.id }, "speech interrupted by VAD");
-      (_b = this.realtimeSession) == null ? void 0 : _b.interrupt();
+      this.logger.info(
+        { "speech id": this._currentSpeech.id },
+        "speech interrupted by audio activity"
+      );
+      (_d = this.realtimeSession) == null ? void 0 : _d.interrupt();
       this._currentSpeech.interrupt();
     }
   }
+  onInterruption(ev) {
+    this.restoreInterruptionByAudioActivity();
+    this.interruptByAudioActivity();
+    if (this.audioRecognition) {
+      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt);
+    }
+  }
   onInterimTranscript(ev) {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
       return;
@@ -462,10 +697,14 @@ class AgentActivity {
       AgentSessionEventTypes.UserInputTranscribed,
       createUserInputTranscribedEvent({
         transcript: ev.alternatives[0].text,
-        isFinal: false
+        isFinal: false,
+        language: ev.alternatives[0].language
         // TODO(AJS-106): add multi participant support
       })
     );
+    if (ev.alternatives[0].text) {
+      this.interruptByAudioActivity();
+    }
   }
   onFinalTranscript(ev) {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
@@ -475,13 +714,70 @@ class AgentActivity {
       AgentSessionEventTypes.UserInputTranscribed,
       createUserInputTranscribedEvent({
         transcript: ev.alternatives[0].text,
-        isFinal: true
+        isFinal: true,
+        language: ev.alternatives[0].language
         // TODO(AJS-106): add multi participant support
       })
     );
+    if (this.audioRecognition && this.turnDetection !== "manual" && this.turnDetection !== "realtime_llm") {
+      this.interruptByAudioActivity();
+    }
+  }
+  onPreemptiveGeneration(info) {
+    if (!this.agentSession.sessionOptions.preemptiveGeneration || this.schedulingPaused || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof LLM)) {
+      return;
+    }
+    this.cancelPreemptiveGeneration();
+    this.logger.info(
+      {
+        newTranscript: info.newTranscript,
+        transcriptConfidence: info.transcriptConfidence
+      },
+      "starting preemptive generation"
+    );
+    const userMessage = ChatMessage.create({
+      role: "user",
+      content: info.newTranscript,
+      transcriptConfidence: info.transcriptConfidence
+    });
+    const chatCtx = this.agent.chatCtx.copy();
+    const speechHandle = this.generateReply({
+      userMessage,
+      chatCtx,
+      scheduleSpeech: false
+    });
+    this._preemptiveGeneration = {
+      speechHandle,
+      userMessage,
+      info,
+      chatCtx: chatCtx.copy(),
+      tools: { ...this.tools },
+      toolChoice: this.toolChoice,
+      createdAt: Date.now()
+    };
+  }
+  cancelPreemptiveGeneration() {
+    if (this._preemptiveGeneration !== void 0) {
+      this._preemptiveGeneration.speechHandle._cancel();
+      this._preemptiveGeneration = void 0;
+    }
   }
   createSpeechTask(options) {
-    const { task, ownedSpeechHandle } = options;
+    const { taskFn, controller, ownedSpeechHandle, inlineTask, name } = options;
+    const wrappedFn = (ctrl) => {
+      return agentActivityStorage.run(this, () => {
+        const currentTask = Task.current();
+        if (currentTask) {
+          _setActivityTaskInfo(currentTask, { speechHandle: ownedSpeechHandle, inlineTask });
+        }
+        if (ownedSpeechHandle) {
+          return speechHandleStorage.run(ownedSpeechHandle, () => taskFn(ctrl));
+        }
+        return taskFn(ctrl);
+      });
+    };
+    const task = Task.from(wrappedFn, controller, name);
+    _setActivityTaskInfo(task, { speechHandle: ownedSpeechHandle, inlineTask });
     this.speechTasks.add(task);
     task.addDoneCallback(() => {
       this.speechTasks.delete(task);
@@ -497,20 +793,35 @@ class AgentActivity {
     task.addDoneCallback(() => {
       this.wakeupMainTask();
     });
-    return task.result;
+    return task;
   }
   async onEndOfTurn(info) {
-    if (this.draining) {
-      this.logger.warn({ user_input: info.newTranscript }, "skipping user input, task is draining");
+    var _a, _b;
+    if (this.schedulingPaused) {
+      this.cancelPreemptiveGeneration();
+      this.logger.warn(
+        { user_input: info.newTranscript },
+        "skipping user input, speech scheduling is paused"
+      );
       return true;
     }
-    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && this.agentSession.options.minInterruptionWords > 0 && info.newTranscript.split(" ").length < this.agentSession.options.minInterruptionWords) {
-      this.logger.info("skipping user input, new_transcript is too short");
-      return false;
+    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && ((_a = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0) {
+      const wordCount = splitWords(info.newTranscript, true).length;
+      if (wordCount < ((_b = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
+        this.cancelPreemptiveGeneration();
+        this.logger.info(
+          {
+            wordCount,
+            minInterruptionWords: this.agentSession.sessionOptions.turnHandling.interruption.minWords
+          },
+          "skipping user input, word count below minimum interruption threshold"
+        );
+        return false;
+      }
     }
     const oldTask = this._userTurnCompletedTask;
     this._userTurnCompletedTask = this.createSpeechTask({
-      task: Task.from(() => this.userTurnCompleted(info, oldTask)),
+      taskFn: () => this.userTurnCompleted(info, oldTask),
       name: "AgentActivity.userTurnCompleted"
     });
     return true;
@@ -535,19 +846,49 @@ class AgentActivity {
           throw new Error("Speech queue is empty");
         }
         const speechHandle = heapItem[2];
+        if (speechHandle.interrupted || speechHandle.done()) {
+          continue;
+        }
         this._currentSpeech = speechHandle;
         speechHandle._authorizeGeneration();
-        await speechHandle._waitForGeneration();
+        await speechHandle.waitIfNotInterrupted([speechHandle._waitForGeneration()]);
         this._currentSpeech = void 0;
       }
-      if (this.draining && this.speechTasks.size === 0) {
-        this.logger.info("mainTask: draining and no more speech tasks");
+      const toWait = this.getDrainPendingSpeechTasks();
+      if (this._schedulingPaused && toWait.length === 0) {
+        this.logger.info("mainTask: scheduling paused and no more speech tasks to wait");
         break;
       }
       this.q_updated = new Future();
     }
     this.logger.info("AgentActivity mainTask: exiting");
   }
+  getDrainPendingSpeechTasks() {
+    const blockedHandles = [];
+    for (const task of this._drainBlockedTasks) {
+      const info = _getActivityTaskInfo(task);
+      if (!info) {
+        this.logger.error("blocked task without activity info; skipping.");
+        continue;
+      }
+      if (!info.speechHandle) {
+        continue;
+      }
+      blockedHandles.push(info.speechHandle);
+    }
+    const toWait = [];
+    for (const task of this.speechTasks) {
+      if (this._drainBlockedTasks.includes(task)) {
+        continue;
+      }
+      const info = _getActivityTaskInfo(task);
+      if (info && info.speechHandle && blockedHandles.includes(info.speechHandle)) {
+        continue;
+      }
+      toWait.push(task);
+    }
+    return toWait;
+  }
   wakeupMainTask() {
     this.q_updated.resolve();
   }
@@ -558,7 +899,8 @@ class AgentActivity {
       chatCtx,
       instructions: defaultInstructions,
       toolChoice: defaultToolChoice,
-      allowInterruptions: defaultAllowInterruptions
+      allowInterruptions: defaultAllowInterruptions,
+      scheduleSpeech = true
     } = options;
     let instructions = defaultInstructions;
     let toolChoice = defaultToolChoice;
@@ -572,7 +914,7 @@ class AgentActivity {
     if (this.llm === void 0) {
       throw new Error("trying to generate reply without an LLM model");
     }
-    const functionCall = (_a = asyncLocalStorage.getStore()) == null ? void 0 : _a.functionCall;
+    const functionCall = (_a = functionCallStorage.getStore()) == null ? void 0 : _a.functionCall;
     if (toolChoice === void 0 && functionCall !== void 0) {
       toolChoice = "none";
     }
@@ -590,19 +932,17 @@ class AgentActivity {
     this.logger.info({ speech_id: handle.id }, "Creating speech handle");
     if (this.llm instanceof RealtimeModel) {
       this.createSpeechTask({
-        task: Task.from(
-          (abortController) => this.realtimeReplyTask({
-            speechHandle: handle,
-            // TODO(brian): support llm.ChatMessage for the realtime model
-            userInput: userMessage == null ? void 0 : userMessage.textContent,
-            instructions,
-            modelSettings: {
-              // isGiven(toolChoice) = toolChoice !== undefined
-              toolChoice: toOaiToolChoice(toolChoice !== void 0 ? toolChoice : this.toolChoice)
-            },
-            abortController
-          })
-        ),
+        taskFn: (abortController) => this.realtimeReplyTask({
+          speechHandle: handle,
+          // TODO(brian): support llm.ChatMessage for the realtime model
+          userInput: userMessage == null ? void 0 : userMessage.textContent,
+          instructions,
+          modelSettings: {
+            // isGiven(toolChoice) = toolChoice !== undefined
+            toolChoice: toOaiToolChoice(toolChoice !== void 0 ? toolChoice : this.toolChoice)
+          },
+          abortController
+        }),
         ownedSpeechHandle: handle,
         name: "AgentActivity.realtimeReply"
       });
@@ -611,39 +951,56 @@ class AgentActivity {
         instructions = `${this.agent.instructions}
 ${instructions}`;
       }
+      const onEnterData = onEnterStorage.getStore();
+      const shouldFilterTools = (onEnterData == null ? void 0 : onEnterData.agent) === this.agent && (onEnterData == null ? void 0 : onEnterData.session) === this.agentSession;
+      const tools = shouldFilterTools ? Object.fromEntries(
+        Object.entries(this.agent.toolCtx).filter(
+          ([, fnTool]) => !(fnTool.flags & ToolFlag.IGNORE_ON_ENTER)
+        )
+      ) : this.agent.toolCtx;
       const task = this.createSpeechTask({
-        task: Task.from(
-          (abortController) => this.pipelineReplyTask(
-            handle,
-            chatCtx ?? this.agent.chatCtx,
-            this.agent.toolCtx,
-            {
-              toolChoice: toOaiToolChoice(toolChoice !== void 0 ? toolChoice : this.toolChoice)
-            },
-            abortController,
-            instructions ? `${this.agent.instructions}
-${instructions}` : instructions,
-            userMessage
-          )
+        taskFn: (abortController) => this.pipelineReplyTask(
+          handle,
+          chatCtx ?? this.agent.chatCtx,
+          tools,
+          {
+            toolChoice: toOaiToolChoice(toolChoice !== void 0 ? toolChoice : this.toolChoice)
+          },
+          abortController,
+          instructions,
+          userMessage
         ),
         ownedSpeechHandle: handle,
         name: "AgentActivity.pipelineReply"
       });
-      task.finally(() => this.onPipelineReplyDone());
+      task.result.finally(() => this.onPipelineReplyDone());
+    }
+    if (scheduleSpeech) {
+      this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
     }
-    this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
     return handle;
   }
-  interrupt() {
+  interrupt(options = {}) {
     var _a;
+    const { force = false } = options;
+    this.cancelPreemptiveGeneration();
     const future = new Future();
     const currentSpeech = this._currentSpeech;
-    currentSpeech == null ? void 0 : currentSpeech.interrupt();
+    currentSpeech == null ? void 0 : currentSpeech.interrupt(force);
     for (const [_, __, speech] of this.speechQueue) {
-      speech.interrupt();
+      speech.interrupt(force);
     }
     (_a = this.realtimeSession) == null ? void 0 : _a.interrupt();
-    if (currentSpeech === void 0) {
+    if (force) {
+      for (const task of this.speechTasks) {
+        task.cancel();
+      }
+      if (currentSpeech && !currentSpeech.done()) {
+        currentSpeech._markDone();
+      }
+      this.speechQueue.clear();
+      future.resolve();
+    } else if (currentSpeech === void 0) {
       future.resolve();
     } else {
       currentSpeech.addDoneCallback(() => {
@@ -661,7 +1018,7 @@ ${instructions}` : instructions,
   async userTurnCompleted(info, oldTask) {
     var _a, _b;
     if (oldTask) {
-      await oldTask;
+      await oldTask.result;
     }
     if (this.llm instanceof RealtimeModel) {
       if (this.llm.capabilities.turnDetection) {
@@ -686,7 +1043,8 @@ ${instructions}` : instructions,
     }
     let userMessage = ChatMessage.create({
       role: "user",
-      content: info.newTranscript
+      content: info.newTranscript,
+      transcriptConfidence: info.transcriptConfidence
     });
     const chatCtx = this.agent.chatCtx.copy();
     const startTime = Date.now();
@@ -704,13 +1062,57 @@ ${instructions}` : instructions,
     } else if (this.llm === void 0) {
       return;
     }
-    const speechHandle = this.generateReply({ userMessage, chatCtx });
+    const userMetricsReport = {};
+    if (info.startedSpeakingAt !== void 0) {
+      userMetricsReport.startedSpeakingAt = info.startedSpeakingAt / 1e3;
+    }
+    if (info.stoppedSpeakingAt !== void 0) {
+      userMetricsReport.stoppedSpeakingAt = info.stoppedSpeakingAt / 1e3;
+    }
+    if (info.transcriptionDelay !== void 0) {
+      userMetricsReport.transcriptionDelay = info.transcriptionDelay / 1e3;
+    }
+    if (info.endOfUtteranceDelay !== void 0) {
+      userMetricsReport.endOfTurnDelay = info.endOfUtteranceDelay / 1e3;
+    }
+    userMetricsReport.onUserTurnCompletedDelay = callbackDuration / 1e3;
+    if (userMessage) {
+      userMessage.metrics = userMetricsReport;
+    }
+    let speechHandle;
+    if (this._preemptiveGeneration !== void 0) {
+      const preemptive = this._preemptiveGeneration;
+      if (preemptive.info.newTranscript === (userMessage == null ? void 0 : userMessage.textContent) && preemptive.chatCtx.isEquivalent(chatCtx) && isSameToolContext(preemptive.tools, this.tools) && isSameToolChoice(preemptive.toolChoice, this.toolChoice)) {
+        speechHandle = preemptive.speechHandle;
+        if (preemptive.userMessage && userMessage) {
+          preemptive.userMessage.metrics = userMetricsReport;
+          preemptive.userMessage.transcriptConfidence = userMessage.transcriptConfidence;
+        }
+        this.scheduleSpeech(speechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
+        this.logger.debug(
+          {
+            preemptiveLeadTime: Date.now() - preemptive.createdAt
+          },
+          "using preemptive generation"
+        );
+      } else {
+        this.logger.warn(
+          "preemptive generation enabled but chat context or tools have changed after `onUserTurnCompleted`"
+        );
+        preemptive.speechHandle._cancel();
+      }
+      this._preemptiveGeneration = void 0;
+    }
+    if (speechHandle === void 0) {
+      speechHandle = this.generateReply({ userMessage, chatCtx });
+    }
     const eouMetrics = {
       type: "eou_metrics",
       timestamp: Date.now(),
-      endOfUtteranceDelay: info.endOfUtteranceDelay,
-      transcriptionDelay: info.transcriptionDelay,
-      onUserTurnCompletedDelay: callbackDuration,
+      endOfUtteranceDelayMs: info.endOfUtteranceDelay,
+      transcriptionDelayMs: info.transcriptionDelay,
+      onUserTurnCompletedDelayMs: callbackDuration,
+      lastSpeakingTimeMs: info.stoppedSpeakingAt ?? 0,
       speechId: speechHandle.id
     };
     this.agentSession.emit(
@@ -719,6 +1121,8 @@ ${instructions}` : instructions,
     );
   }
   async ttsTask(speechHandle, text, addToChatCtx, modelSettings, replyAbortController, audio) {
+    var _a, _b;
+    speechHandle._agentTurnContext = otelContext.active();
     speechHandleStorage.enterWith(speechHandle);
     const transcriptionOutput = this.agentSession.output.transcriptionEnabled ? this.agentSession.output.transcription : null;
     const audioOutput = this.agentSession.output.audioEnabled ? this.agentSession.output.audio : null;
@@ -750,25 +1154,38 @@ ${instructions}` : instructions,
       textOut = _textOut;
       tasks.push(textForwardTask);
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    let replyStartedSpeakingAt;
+    let replyTtsGenData = null;
+    const onFirstFrame = (startedSpeakingAt) => {
+      replyStartedSpeakingAt = startedSpeakingAt ?? Date.now();
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfAgentSpeech();
+        this.isInterruptionByAudioActivityEnabled = false;
+      }
     };
     if (!audioOutput) {
       if (textOut) {
-        textOut.firstTextFut.await.finally(onFirstFrame);
+        textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
       }
     } else {
       let audioOut = null;
       if (!audio) {
-        const [ttsTask, ttsStream] = performTTSInference(
+        const [ttsTask, ttsGenData] = performTTSInference(
           (...args) => this.agent.ttsNode(...args),
           audioSource,
           modelSettings,
-          replyAbortController
+          replyAbortController,
+          (_a = this.tts) == null ? void 0 : _a.model,
+          (_b = this.tts) == null ? void 0 : _b.provider
         );
         tasks.push(ttsTask);
+        replyTtsGenData = ttsGenData;
         const [forwardTask, _audioOut] = performAudioForwarding(
-          ttsStream,
+          ttsGenData.audioStream,
           audioOutput,
           replyAbortController
         );
@@ -783,7 +1200,7 @@ ${instructions}` : instructions,
         tasks.push(forwardTask);
         audioOut = _audioOut;
       }
-      audioOut.firstFrameFut.await.finally(onFirstFrame);
+      audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
     }
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
@@ -798,28 +1215,63 @@ ${instructions}` : instructions,
       }
     }
     if (addToChatCtx) {
+      const replyStoppedSpeakingAt = Date.now();
+      const replyAssistantMetrics = {};
+      if ((replyTtsGenData == null ? void 0 : replyTtsGenData.ttfb) !== void 0) {
+        replyAssistantMetrics.ttsNodeTtfb = replyTtsGenData.ttfb;
+      }
+      if (replyStartedSpeakingAt !== void 0) {
+        replyAssistantMetrics.startedSpeakingAt = replyStartedSpeakingAt / 1e3;
+        replyAssistantMetrics.stoppedSpeakingAt = replyStoppedSpeakingAt / 1e3;
+      }
       const message = ChatMessage.create({
         role: "assistant",
         content: (textOut == null ? void 0 : textOut.text) || "",
-        interrupted: speechHandle.interrupted
+        interrupted: speechHandle.interrupted,
+        metrics: replyAssistantMetrics
       });
       this.agent._chatCtx.insert(message);
       this.agentSession._conversationItemAdded(message);
     }
     if (this.agentSession.agentState === "speaking") {
       this.agentSession._updateAgentState("listening");
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onEndOfAgentSpeech(Date.now());
+      }
+      this.restoreInterruptionByAudioActivity();
     }
   }
-  async pipelineReplyTask(speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages) {
-    var _a, _b, _c;
+  _pipelineReplyTaskImpl = async ({
+    speechHandle,
+    chatCtx,
+    toolCtx,
+    modelSettings,
+    replyAbortController,
+    instructions,
+    newMessage,
+    toolsMessages,
+    span,
+    _previousUserMetrics
+  }) => {
+    var _a, _b, _c, _d, _e, _f;
+    speechHandle._agentTurnContext = otelContext.active();
+    span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
+    if (instructions) {
+      span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, instructions);
+    }
+    if (newMessage) {
+      span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || "");
+    }
+    const localParticipant = (_a = this.agentSession._roomIO) == null ? void 0 : _a.localParticipant;
+    if (localParticipant) {
+      setParticipantSpanAttributes(span, localParticipant);
+    }
     speechHandleStorage.enterWith(speechHandle);
     const audioOutput = this.agentSession.output.audioEnabled ? this.agentSession.output.audio : null;
     const transcriptionOutput = this.agentSession.output.transcriptionEnabled ? this.agentSession.output.transcription : null;
     chatCtx = chatCtx.copy();
     if (newMessage) {
       chatCtx.insert(newMessage);
-      this.agent._chatCtx.insert(newMessage);
-      this.agentSession._conversationItemAdded(newMessage);
     }
     if (instructions) {
       try {
@@ -832,7 +1284,6 @@ ${instructions}` : instructions,
         this.logger.error({ error: e }, "error occurred during updateInstructions");
       }
     }
-    this.agentSession._updateAgentState("thinking");
     const tasks = [];
     const [llmTask, llmGenData] = performLLMInference(
       // preserve  `this` context in llmNode
@@ -840,22 +1291,36 @@ ${instructions}` : instructions,
       chatCtx,
       toolCtx,
       modelSettings,
-      replyAbortController
+      replyAbortController,
+      (_b = this.llm) == null ? void 0 : _b.model,
+      (_c = this.llm) == null ? void 0 : _c.provider
     );
     tasks.push(llmTask);
-    const [ttsTextInput, llmOutput] = llmGenData.textStream.tee();
     let ttsTask = null;
-    let ttsStream = null;
+    let ttsGenData = null;
+    let llmOutput;
     if (audioOutput) {
-      [ttsTask, ttsStream] = performTTSInference(
+      const [ttsTextInput, textOutput] = llmGenData.textStream.tee();
+      llmOutput = textOutput;
+      [ttsTask, ttsGenData] = performTTSInference(
         (...args) => this.agent.ttsNode(...args),
         ttsTextInput,
         modelSettings,
-        replyAbortController
+        replyAbortController,
+        (_d = this.tts) == null ? void 0 : _d.model,
+        (_e = this.tts) == null ? void 0 : _e.provider
       );
       tasks.push(ttsTask);
+    } else {
+      llmOutput = llmGenData.textStream;
     }
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForScheduled()]);
+    let userMetrics = _previousUserMetrics;
+    if (newMessage && speechHandle.scheduled) {
+      this.agent._chatCtx.insert(newMessage);
+      this.agentSession._conversationItemAdded(newMessage);
+      userMetrics = newMessage.metrics;
+    }
     if (speechHandle.interrupted) {
       replyAbortController.abort();
       await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
@@ -865,7 +1330,20 @@ ${instructions}` : instructions,
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForAuthorization()]);
     speechHandle._clearAuthorization();
     const replyStartedAt = Date.now();
-    const trNodeResult = await this.agent.transcriptionNode(llmOutput, modelSettings);
+    let transcriptionInput = llmOutput;
+    if (this.useTtsAlignedTranscript && ((_f = this.tts) == null ? void 0 : _f.capabilities.alignedTranscript) && ttsGenData) {
+      const timedTextsStream = await Promise.race([
+        ttsGenData.timedTextsFut.await,
+        (ttsTask == null ? void 0 : ttsTask.result.catch(
+          () => this.logger.warn("TTS task failed before resolving timedTextsFut")
+        )) ?? Promise.resolve()
+      ]);
+      if (timedTextsStream) {
+        this.logger.debug("Using TTS aligned transcripts for transcription node input");
+        transcriptionInput = timedTextsStream;
+      }
+    }
+    const trNodeResult = await this.agent.transcriptionNode(transcriptionInput, modelSettings);
     let textOut = null;
     if (trNodeResult) {
       const [textForwardTask, _textOut] = performTextForwarding(
@@ -876,29 +1354,44 @@ ${instructions}` : instructions,
       tasks.push(textForwardTask);
       textOut = _textOut;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    let agentStartedSpeakingAt;
+    const onFirstFrame = (startedSpeakingAt) => {
+      agentStartedSpeakingAt = startedSpeakingAt ?? Date.now();
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfAgentSpeech();
+        this.isInterruptionByAudioActivityEnabled = false;
+      }
     };
     let audioOut = null;
     if (audioOutput) {
-      if (ttsStream) {
+      if (ttsGenData) {
         const [forwardTask, _audioOut] = performAudioForwarding(
-          ttsStream,
+          ttsGenData.audioStream,
           audioOutput,
           replyAbortController
         );
         audioOut = _audioOut;
         tasks.push(forwardTask);
-        audioOut.firstFrameFut.await.finally(onFirstFrame);
+        audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
       } else {
-        throw Error("ttsStream is null when audioOutput is enabled");
+        throw Error("ttsGenData is null when audioOutput is enabled");
       }
     } else {
-      textOut == null ? void 0 : textOut.firstTextFut.await.finally(onFirstFrame);
+      textOut == null ? void 0 : textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
     }
-    const onToolExecutionStarted = (_) => {
+    const onToolExecutionStarted = (f) => {
+      speechHandle._itemAdded([f]);
+      this.agent._chatCtx.items.push(f);
+      this.agentSession._toolItemsAdded([f]);
     };
-    const onToolExecutionCompleted = (_) => {
+    const onToolExecutionCompleted = (out) => {
+      if (out.toolCallOutput) {
+        speechHandle._itemAdded([out.toolCallOutput]);
+      }
     };
     const [executeToolsTask, toolOutput] = performToolExecutions({
       session: this.agentSession,
@@ -914,28 +1407,53 @@ ${instructions}` : instructions,
     if (audioOutput) {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
     }
+    const agentStoppedSpeakingAt = Date.now();
+    const assistantMetrics = {};
+    if (llmGenData.ttft !== void 0) {
+      assistantMetrics.llmNodeTtft = llmGenData.ttft;
+    }
+    if ((ttsGenData == null ? void 0 : ttsGenData.ttfb) !== void 0) {
+      assistantMetrics.ttsNodeTtfb = ttsGenData.ttfb;
+    }
+    if (agentStartedSpeakingAt !== void 0) {
+      assistantMetrics.startedSpeakingAt = agentStartedSpeakingAt / 1e3;
+      assistantMetrics.stoppedSpeakingAt = agentStoppedSpeakingAt / 1e3;
+      if ((userMetrics == null ? void 0 : userMetrics.stoppedSpeakingAt) !== void 0) {
+        const e2eLatency = agentStartedSpeakingAt / 1e3 - userMetrics.stoppedSpeakingAt;
+        assistantMetrics.e2eLatency = e2eLatency;
+        span.setAttribute(traceTypes.ATTR_E2E_LATENCY, e2eLatency);
+      }
+    }
+    span.setAttribute(traceTypes.ATTR_SPEECH_INTERRUPTED, speechHandle.interrupted);
+    let hasSpeechMessage = false;
     if (toolsMessages) {
       for (const msg of toolsMessages) {
         msg.createdAt = replyStartedAt;
       }
-      this.agent._chatCtx.insert(toolsMessages);
+      const toolCallOutputs = toolsMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agent._chatCtx.insert(toolCallOutputs);
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
         { speech_id: speechHandle.id },
         "Aborting all pipeline reply tasks due to interruption"
       );
+      if (audioOutput) {
+        audioOutput.clearBuffer();
+      }
       replyAbortController.abort();
-      await Promise.allSettled(
-        tasks.map((task) => task.cancelAndWait(AgentActivity.REPLY_TASK_CANCEL_TIMEOUT))
-      );
+      await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
       let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
       if (audioOutput) {
-        audioOutput.clearBuffer();
         const playbackEv = await audioOutput.waitForPlayout();
-        if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+        if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
           this.logger.info(
-            { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+            { speech_id: speechHandle.id, playbackPositionInS: playbackEv.playbackPosition },
             "playout interrupted"
           );
           if (playbackEv.synchronizedTranscript) {
@@ -946,19 +1464,27 @@ ${instructions}` : instructions,
         }
       }
       if (forwardedText) {
+        hasSpeechMessage = true;
         const message = ChatMessage.create({
           role: "assistant",
           content: forwardedText,
           id: llmGenData.id,
           interrupted: true,
-          createdAt: replyStartedAt
+          createdAt: replyStartedAt,
+          metrics: assistantMetrics
         });
         chatCtx.insert(message);
         this.agent._chatCtx.insert(message);
+        speechHandle._itemAdded([message]);
         this.agentSession._conversationItemAdded(message);
+        span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, forwardedText);
       }
       if (this.agentSession.agentState === "speaking") {
         this.agentSession._updateAgentState("listening");
+        if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+          this.restoreInterruptionByAudioActivity();
+        }
       }
       this.logger.info(
         { speech_id: speechHandle.id, message: forwardedText },
@@ -969,16 +1495,20 @@ ${instructions}` : instructions,
       return;
     }
     if (textOut && textOut.text) {
+      hasSpeechMessage = true;
       const message = ChatMessage.create({
         role: "assistant",
         id: llmGenData.id,
         interrupted: false,
         createdAt: replyStartedAt,
-        content: textOut.text
+        content: textOut.text,
+        metrics: assistantMetrics
       });
       chatCtx.insert(message);
       this.agent._chatCtx.insert(message);
+      speechHandle._itemAdded([message]);
       this.agentSession._conversationItemAdded(message);
+      span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, textOut.text);
       this.logger.info(
         { speech_id: speechHandle.id, message: textOut.text },
         "playout completed without interruption"
@@ -988,11 +1518,17 @@ ${instructions}` : instructions,
       this.agentSession._updateAgentState("thinking");
     } else if (this.agentSession.agentState === "speaking") {
       this.agentSession._updateAgentState("listening");
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+          this.restoreInterruptionByAudioActivity();
+        }
+      }
     }
     speechHandle._markGenerationDone();
     await executeToolsTask.result;
     if (toolOutput.output.length === 0) return;
-    const { maxToolSteps } = this.agentSession.options;
+    const { maxToolSteps } = this.agentSession.sessionOptions;
     if (speechHandle.numSteps >= maxToolSteps) {
       this.logger.warn(
         { speech_id: speechHandle.id, max_tool_steps: maxToolSteps },
@@ -1000,45 +1536,15 @@ ${instructions}` : instructions,
       );
       return;
     }
-    const functionToolsExecutedEvent = createFunctionToolsExecutedEvent({
-      functionCalls: [],
-      functionCallOutputs: []
-    });
-    let shouldGenerateToolReply = false;
-    let newAgentTask = null;
-    let ignoreTaskSwitch = false;
-    for (const sanitizedOut of toolOutput.output) {
-      if (sanitizedOut.toolCallOutput !== void 0) {
-        functionToolsExecutedEvent.functionCalls.push(sanitizedOut.toolCall);
-        functionToolsExecutedEvent.functionCallOutputs.push(sanitizedOut.toolCallOutput);
-        if (sanitizedOut.replyRequired) {
-          shouldGenerateToolReply = true;
-        }
-      }
-      if (newAgentTask !== null && sanitizedOut.agentTask !== void 0) {
-        this.logger.error("expected to receive only one agent task from the tool executions");
-        ignoreTaskSwitch = true;
-      }
-      newAgentTask = sanitizedOut.agentTask ?? null;
-      this.logger.debug(
-        {
-          speechId: speechHandle.id,
-          name: (_a = sanitizedOut.toolCall) == null ? void 0 : _a.name,
-          args: sanitizedOut.toolCall.args,
-          output: (_b = sanitizedOut.toolCallOutput) == null ? void 0 : _b.output,
-          isError: (_c = sanitizedOut.toolCallOutput) == null ? void 0 : _c.isError
-        },
-        "Tool call execution finished"
-      );
-    }
+    const { functionToolsExecutedEvent, shouldGenerateToolReply, newAgentTask, ignoreTaskSwitch } = this.summarizeToolExecutionOutput(toolOutput, speechHandle);
     this.agentSession.emit(
       AgentSessionEventTypes.FunctionToolsExecuted,
       functionToolsExecutedEvent
     );
-    let draining = this.draining;
+    let schedulingPaused = this.schedulingPaused;
     if (!ignoreTaskSwitch && newAgentTask !== null) {
       this.agentSession.updateAgent(newAgentTask);
-      draining = true;
+      schedulingPaused = true;
     }
     const toolMessages = [
       ...functionToolsExecutedEvent.functionCalls,
@@ -1046,47 +1552,85 @@ ${instructions}` : instructions,
     ];
     if (shouldGenerateToolReply) {
       chatCtx.insert(toolMessages);
-      const handle = SpeechHandle.create({
-        allowInterruptions: speechHandle.allowInterruptions,
-        stepIndex: speechHandle._stepIndex + 1,
-        parent: speechHandle
-      });
-      this.agentSession.emit(
-        AgentSessionEventTypes.SpeechCreated,
-        createSpeechCreatedEvent({
-          userInitiated: false,
-          source: "tool_response",
-          speechHandle: handle
-        })
-      );
-      const respondToolChoice = draining || modelSettings.toolChoice === "none" ? "none" : "auto";
+      speechHandle._numSteps += 1;
+      const respondToolChoice = schedulingPaused || modelSettings.toolChoice === "none" ? "none" : "auto";
       const toolResponseTask = this.createSpeechTask({
-        task: Task.from(
-          () => this.pipelineReplyTask(
-            handle,
-            chatCtx,
-            toolCtx,
-            { toolChoice: respondToolChoice },
-            replyAbortController,
-            instructions,
-            void 0,
-            toolMessages
-          )
+        taskFn: () => this.pipelineReplyTask(
+          speechHandle,
+          chatCtx,
+          toolCtx,
+          { toolChoice: respondToolChoice },
+          replyAbortController,
+          instructions,
+          void 0,
+          toolMessages,
+          hasSpeechMessage ? void 0 : userMetrics
         ),
-        ownedSpeechHandle: handle,
+        ownedSpeechHandle: speechHandle,
         name: "AgentActivity.pipelineReply"
       });
-      toolResponseTask.finally(() => this.onPipelineReplyDone());
-      this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL, true);
+      toolResponseTask.result.finally(() => this.onPipelineReplyDone());
+      this.scheduleSpeech(speechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL, true);
     } else if (functionToolsExecutedEvent.functionCallOutputs.length > 0) {
       for (const msg of toolMessages) {
         msg.createdAt = replyStartedAt;
       }
-      this.agent._chatCtx.insert(toolMessages);
+      const toolCallOutputs = toolMessages.filter(
+        (m) => m.type === "function_call_output"
+      );
+      if (toolCallOutputs.length > 0) {
+        this.agent._chatCtx.insert(toolCallOutputs);
+        this.agentSession._toolItemsAdded(toolCallOutputs);
+      }
     }
-  }
+  };
+  pipelineReplyTask = async (speechHandle, chatCtx, toolCtx, modelSettings, replyAbortController, instructions, newMessage, toolsMessages, _previousUserMetrics) => tracer.startActiveSpan(
+    async (span) => this._pipelineReplyTaskImpl({
+      speechHandle,
+      chatCtx,
+      toolCtx,
+      modelSettings,
+      replyAbortController,
+      instructions,
+      newMessage,
+      toolsMessages,
+      span,
+      _previousUserMetrics
+    }),
+    {
+      name: "agent_turn",
+      context: this.agentSession.rootSpanContext
+    }
+  );
   async realtimeGenerationTask(speechHandle, ev, modelSettings, replyAbortController) {
-    var _a, _b, _c;
+    return tracer.startActiveSpan(
+      async (span) => this._realtimeGenerationTaskImpl({
+        speechHandle,
+        ev,
+        modelSettings,
+        replyAbortController,
+        span
+      }),
+      {
+        name: "agent_turn",
+        context: this.agentSession.rootSpanContext
+      }
+    );
+  }
+  async _realtimeGenerationTaskImpl({
+    speechHandle,
+    ev,
+    modelSettings,
+    replyAbortController,
+    span
+  }) {
+    var _a;
+    speechHandle._agentTurnContext = otelContext.active();
+    span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
+    const localParticipant = (_a = this.agentSession._roomIO) == null ? void 0 : _a.localParticipant;
+    if (localParticipant) {
+      setParticipantSpanAttributes(span, localParticipant);
+    }
     speechHandleStorage.enterWith(speechHandle);
     if (!this.realtimeSession) {
       throw new Error("realtime session is not initialized");
@@ -1094,6 +1638,10 @@ ${instructions}` : instructions,
     if (!(this.llm instanceof RealtimeModel)) {
       throw new Error("llm is not a realtime model");
     }
+    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.llm.model);
+    if (this.realtimeSpans && ev.responseId) {
+      this.realtimeSpans.set(ev.responseId, span);
+    }
     this.logger.debug(
       { speech_id: speechHandle.id, stepIndex: speechHandle.numSteps },
       "realtime generation started"
@@ -1106,10 +1654,17 @@ ${instructions}` : instructions,
     if (speechHandle.interrupted) {
       return;
     }
-    const onFirstFrame = () => {
-      this.agentSession._updateAgentState("speaking");
+    const onFirstFrame = (startedSpeakingAt) => {
+      this.agentSession._updateAgentState("speaking", {
+        startTime: startedSpeakingAt,
+        otelContext: speechHandle._agentTurnContext
+      });
     };
     const readMessages = async (abortController, outputs) => {
+      var _a2, _b;
+      replyAbortController.signal.addEventListener("abort", () => abortController.abort(), {
+        once: true
+      });
       const forwardTasks = [];
       try {
         for await (const msg of ev.messageStream) {
@@ -1119,7 +1674,22 @@ ${instructions}` : instructions,
             );
             break;
           }
-          const trNodeResult = await this.agent.transcriptionNode(msg.textStream, modelSettings);
+          const msgModalities = msg.modalities ? await msg.modalities : void 0;
+          let ttsTextInput = null;
+          let trTextInput;
+          if (msgModalities && !msgModalities.includes("audio") && this.tts) {
+            if (this.llm instanceof RealtimeModel && this.llm.capabilities.audioOutput) {
+              this.logger.warn(
+                "text response received from realtime API, falling back to use a TTS model."
+              );
+            }
+            const [_ttsTextInput, _trTextInput] = msg.textStream.tee();
+            ttsTextInput = _ttsTextInput;
+            trTextInput = _trTextInput;
+          } else {
+            trTextInput = msg.textStream;
+          }
+          const trNodeResult = await this.agent.transcriptionNode(trTextInput, modelSettings);
           let textOut = null;
           if (trNodeResult) {
             const [textForwardTask, _textOut] = performTextForwarding(
@@ -1132,28 +1702,46 @@ ${instructions}` : instructions,
           }
           let audioOut = null;
           if (audioOutput) {
-            const realtimeAudio = await this.agent.realtimeAudioOutputNode(
-              msg.audioStream,
-              modelSettings
-            );
-            if (realtimeAudio) {
+            let realtimeAudioResult = null;
+            if (ttsTextInput) {
+              const [ttsTask, ttsGenData] = performTTSInference(
+                (...args) => this.agent.ttsNode(...args),
+                ttsTextInput,
+                modelSettings,
+                abortController,
+                (_a2 = this.tts) == null ? void 0 : _a2.model,
+                (_b = this.tts) == null ? void 0 : _b.provider
+              );
+              tasks.push(ttsTask);
+              realtimeAudioResult = ttsGenData.audioStream;
+            } else if (msgModalities && msgModalities.includes("audio")) {
+              realtimeAudioResult = await this.agent.realtimeAudioOutputNode(
+                msg.audioStream,
+                modelSettings
+              );
+            } else if (this.llm instanceof RealtimeModel && this.llm.capabilities.audioOutput) {
+              this.logger.error(
+                "Text message received from Realtime API with audio modality. This usually happens when text chat context is synced to the API. Try to add a TTS model as fallback or use text modality with TTS instead."
+              );
+            } else {
+              this.logger.warn(
+                "audio output is enabled but neither tts nor realtime audio is available"
+              );
+            }
+            if (realtimeAudioResult) {
               const [forwardTask, _audioOut] = performAudioForwarding(
-                realtimeAudio,
+                realtimeAudioResult,
                 audioOutput,
                 abortController
               );
               forwardTasks.push(forwardTask);
               audioOut = _audioOut;
-              audioOut.firstFrameFut.await.finally(onFirstFrame);
-            } else {
-              this.logger.warn(
-                "audio output is enabled but neither tts nor realtime audio is available"
-              );
+              audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
             }
           } else if (textOut) {
-            textOut.firstTextFut.await.finally(onFirstFrame);
+            textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
           }
-          outputs.push([msg.messageId, textOut, audioOut]);
+          outputs.push([msg.messageId, textOut, audioOut, msgModalities]);
         }
         await waitFor(forwardTasks);
       } catch (error) {
@@ -1166,7 +1754,7 @@ ${instructions}` : instructions,
     const tasks = [
       Task.from(
         (controller) => readMessages(controller, messageOutputs),
-        replyAbortController,
+        void 0,
         "AgentActivity.realtime_generation.read_messages"
       )
     ];
@@ -1194,6 +1782,8 @@ ${instructions}` : instructions,
     );
     const onToolExecutionStarted = (f) => {
       speechHandle._itemAdded([f]);
+      this.agent._chatCtx.items.push(f);
+      this.agentSession._toolItemsAdded([f]);
     };
     const onToolExecutionCompleted = (out) => {
       if (out.toolCallOutput) {
@@ -1213,7 +1803,6 @@ ${instructions}` : instructions,
     await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result));
     if (audioOutput) {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
-      this.agentSession._updateAgentState("listening");
     }
     if (speechHandle.interrupted) {
       this.logger.debug(
@@ -1223,15 +1812,15 @@ ${instructions}` : instructions,
       replyAbortController.abort();
       await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
       if (messageOutputs.length > 0) {
-        const [msgId, textOut, audioOut] = messageOutputs[0];
+        const [msgId, textOut, audioOut, msgModalities] = messageOutputs[0];
         let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
         if (audioOutput) {
           audioOutput.clearBuffer();
           const playbackEv = await audioOutput.waitForPlayout();
-          let playbackPosition = playbackEv.playbackPosition;
-          if (audioOut == null ? void 0 : audioOut.firstFrameFut.done) {
+          let playbackPositionInS = playbackEv.playbackPosition;
+          if ((audioOut == null ? void 0 : audioOut.firstFrameFut.done) && !audioOut.firstFrameFut.rejected) {
             this.logger.info(
-              { speech_id: speechHandle.id, playbackPosition: playbackEv.playbackPosition },
+              { speech_id: speechHandle.id, playbackPositionInS },
               "playout interrupted"
             );
             if (playbackEv.synchronizedTranscript) {
@@ -1239,11 +1828,13 @@ ${instructions}` : instructions,
             }
           } else {
             forwardedText = "";
-            playbackPosition = 0;
+            playbackPositionInS = 0;
           }
           this.realtimeSession.truncate({
             messageId: msgId,
-            audioEndMs: Math.floor(playbackPosition)
+            audioEndMs: Math.floor(playbackPositionInS * 1e3),
+            modalities: msgModalities,
+            audioTranscript: forwardedText
           });
         }
         if (forwardedText) {
@@ -1267,7 +1858,7 @@ ${instructions}` : instructions,
       return;
     }
     if (messageOutputs.length > 0) {
-      const [msgId, textOut, _] = messageOutputs[0];
+      const [msgId, textOut, _, __] = messageOutputs[0];
       const message = ChatMessage.create({
         role: "assistant",
         content: (textOut == null ? void 0 : textOut.text) || "",
@@ -1279,12 +1870,16 @@ ${instructions}` : instructions,
       this.agentSession._conversationItemAdded(message);
     }
     speechHandle._markGenerationDone();
-    toolOutput.firstToolStartedFuture.await.finally(() => {
-      this.agentSession._updateAgentState("thinking");
-    });
     await executeToolsTask.result;
-    if (toolOutput.output.length === 0) return;
-    const { maxToolSteps } = this.agentSession.options;
+    if (toolOutput.output.length > 0) {
+      this.agentSession._updateAgentState("thinking");
+    } else if (this.agentSession.agentState === "speaking") {
+      this.agentSession._updateAgentState("listening");
+    }
+    if (toolOutput.output.length === 0) {
+      return;
+    }
+    const { maxToolSteps } = this.agentSession.sessionOptions;
     if (speechHandle.numSteps >= maxToolSteps) {
       this.logger.warn(
         { speech_id: speechHandle.id, max_tool_steps: maxToolSteps },
@@ -1292,48 +1887,29 @@ ${instructions}` : instructions,
       );
       return;
     }
-    const functionToolsExecutedEvent = createFunctionToolsExecutedEvent({
-      functionCalls: [],
-      functionCallOutputs: []
-    });
-    let shouldGenerateToolReply = false;
-    let newAgentTask = null;
-    let ignoreTaskSwitch = false;
-    for (const sanitizedOut of toolOutput.output) {
-      if (sanitizedOut.toolCallOutput !== void 0) {
-        functionToolsExecutedEvent.functionCallOutputs.push(sanitizedOut.toolCallOutput);
-        if (sanitizedOut.replyRequired) {
-          shouldGenerateToolReply = true;
-        }
-      }
-      if (newAgentTask !== null && sanitizedOut.agentTask !== void 0) {
-        this.logger.error("expected to receive only one agent task from the tool executions");
-        ignoreTaskSwitch = true;
-      }
-      newAgentTask = sanitizedOut.agentTask ?? null;
-      this.logger.debug(
-        {
-          speechId: speechHandle.id,
-          name: (_a = sanitizedOut.toolCall) == null ? void 0 : _a.name,
-          args: sanitizedOut.toolCall.args,
-          output: (_b = sanitizedOut.toolCallOutput) == null ? void 0 : _b.output,
-          isError: (_c = sanitizedOut.toolCallOutput) == null ? void 0 : _c.isError
-        },
-        "Tool call execution finished"
-      );
-    }
+    const { functionToolsExecutedEvent, shouldGenerateToolReply, newAgentTask, ignoreTaskSwitch } = this.summarizeToolExecutionOutput(toolOutput, speechHandle);
     this.agentSession.emit(
       AgentSessionEventTypes.FunctionToolsExecuted,
       functionToolsExecutedEvent
     );
-    let draining = this.draining;
+    let schedulingPaused = this.schedulingPaused;
     if (!ignoreTaskSwitch && newAgentTask !== null) {
       this.agentSession.updateAgent(newAgentTask);
-      draining = true;
+      schedulingPaused = true;
     }
     if (functionToolsExecutedEvent.functionCallOutputs.length > 0) {
+      while (this.currentSpeech || this.speechQueue.size() > 0) {
+        if (this.currentSpeech && !this.currentSpeech.done() && this.currentSpeech !== speechHandle) {
+          await this.currentSpeech.waitForPlayout();
+        } else {
+          await new Promise((resolve) => setImmediate(resolve));
+        }
+      }
       const chatCtx = this.realtimeSession.chatCtx.copy();
       chatCtx.items.push(...functionToolsExecutedEvent.functionCallOutputs);
+      this.agentSession._toolItemsAdded(
+        functionToolsExecutedEvent.functionCallOutputs
+      );
       try {
         await this.realtimeSession.updateChatCtx(chatCtx);
       } catch (error) {
@@ -1360,20 +1936,58 @@ ${instructions}` : instructions,
         speechHandle: replySpeechHandle
       })
     );
-    const toolChoice = draining || modelSettings.toolChoice === "none" ? "none" : "auto";
+    const toolChoice = schedulingPaused || modelSettings.toolChoice === "none" ? "none" : "auto";
     this.createSpeechTask({
-      task: Task.from(
-        (abortController) => this.realtimeReplyTask({
-          speechHandle: replySpeechHandle,
-          modelSettings: { toolChoice },
-          abortController
-        })
-      ),
+      taskFn: (abortController) => this.realtimeReplyTask({
+        speechHandle: replySpeechHandle,
+        modelSettings: { toolChoice },
+        abortController
+      }),
       ownedSpeechHandle: replySpeechHandle,
       name: "AgentActivity.realtime_reply"
     });
     this.scheduleSpeech(replySpeechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL, true);
   }
+  summarizeToolExecutionOutput(toolOutput, speechHandle) {
+    var _a, _b, _c;
+    const functionToolsExecutedEvent = createFunctionToolsExecutedEvent({
+      functionCalls: [],
+      functionCallOutputs: []
+    });
+    let shouldGenerateToolReply = false;
+    let newAgentTask = null;
+    let ignoreTaskSwitch = false;
+    for (const sanitizedOut of toolOutput.output) {
+      if (sanitizedOut.toolCallOutput !== void 0) {
+        functionToolsExecutedEvent.functionCalls.push(sanitizedOut.toolCall);
+        functionToolsExecutedEvent.functionCallOutputs.push(sanitizedOut.toolCallOutput);
+        if (sanitizedOut.replyRequired) {
+          shouldGenerateToolReply = true;
+        }
+      }
+      if (newAgentTask !== null && sanitizedOut.agentTask !== void 0) {
+        this.logger.error("expected to receive only one agent task from the tool executions");
+        ignoreTaskSwitch = true;
+      }
+      newAgentTask = sanitizedOut.agentTask ?? null;
+      this.logger.debug(
+        {
+          speechId: speechHandle.id,
+          name: (_a = sanitizedOut.toolCall) == null ? void 0 : _a.name,
+          args: sanitizedOut.toolCall.args,
+          output: (_b = sanitizedOut.toolCallOutput) == null ? void 0 : _b.output,
+          isError: (_c = sanitizedOut.toolCallOutput) == null ? void 0 : _c.isError
+        },
+        "Tool call execution finished"
+      );
+    }
+    return {
+      functionToolsExecutedEvent,
+      shouldGenerateToolReply,
+      newAgentTask,
+      ignoreTaskSwitch
+    };
+  }
   async realtimeReplyTask({
     speechHandle,
     modelSettings: { toolChoice },
@@ -1415,71 +2029,193 @@ ${instructions}` : instructions,
     }
   }
   scheduleSpeech(speechHandle, priority, force = false) {
-    if (this.draining && !force) {
-      throw new Error("cannot schedule new speech, the agent is draining");
+    if (this.schedulingPaused && !force) {
+      throw new Error("cannot schedule new speech, the speech scheduling is draining/pausing");
     }
     this.speechQueue.push([priority, Number(process.hrtime.bigint()), speechHandle]);
     speechHandle._markScheduled();
     this.wakeupMainTask();
   }
+  async _pauseSchedulingTask(blockedTasks) {
+    if (this._schedulingPaused) return;
+    this._schedulingPaused = true;
+    this._drainBlockedTasks = blockedTasks;
+    this.wakeupMainTask();
+    if (this._mainTask) {
+      await this._mainTask.result;
+    }
+  }
+  _resumeSchedulingTask() {
+    if (!this._schedulingPaused) return;
+    this._schedulingPaused = false;
+    this._mainTask = Task.from(({ signal }) => this.mainTask(signal));
+  }
+  async pause(options = {}) {
+    const { blockedTasks = [] } = options;
+    const unlock = await this.lock.lock();
+    try {
+      const span = tracer.startSpan({
+        name: "pause_agent_activity",
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id }
+      });
+      try {
+        await this._pauseSchedulingTask(blockedTasks);
+        await this._closeSessionResources();
+      } finally {
+        span.end();
+      }
+    } finally {
+      unlock();
+    }
+  }
   async drain() {
-    var _a;
+    return tracer.startActiveSpan(async (span) => this._drainImpl(span), {
+      name: "drain_agent_activity",
+      context: ROOT_CONTEXT
+    });
+  }
+  async _drainImpl(span) {
+    span.setAttribute(traceTypes.ATTR_AGENT_LABEL, this.agent.id);
     const unlock = await this.lock.lock();
     try {
-      if (this._draining) return;
-      this.createSpeechTask({
-        task: Task.from(() => this.agent.onExit()),
+      if (this._schedulingPaused) return;
+      this._onExitTask = this.createSpeechTask({
+        taskFn: () => tracer.startActiveSpan(async () => this.agent.onExit(), {
+          name: "on_exit",
+          attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id }
+        }),
+        inlineTask: true,
         name: "AgentActivity_onExit"
       });
-      this.wakeupMainTask();
-      this._draining = true;
-      await ((_a = this._mainTask) == null ? void 0 : _a.result);
+      this.cancelPreemptiveGeneration();
+      await this._onExitTask.result;
+      await this._pauseSchedulingTask([]);
     } finally {
       unlock();
     }
   }
   async close() {
-    var _a, _b, _c;
     const unlock = await this.lock.lock();
     try {
-      if (!this._draining) {
-        this.logger.warn("task closing without draining");
-      }
-      if (this.llm instanceof LLM) {
-        this.llm.off("metrics_collected", this.onMetricsCollected);
-      }
-      if (this.realtimeSession) {
-        this.realtimeSession.off("generation_created", this.onGenerationCreated);
-        this.realtimeSession.off("input_speech_started", this.onInputSpeechStarted);
-        this.realtimeSession.off("input_speech_stopped", this.onInputSpeechStopped);
-        this.realtimeSession.off(
-          "input_audio_transcription_completed",
-          this.onInputAudioTranscriptionCompleted
-        );
-        this.realtimeSession.off("metrics_collected", this.onMetricsCollected);
+      this.cancelPreemptiveGeneration();
+      await cancelAndWait(Array.from(this.speechTasks), AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
+      if (this._currentSpeech && !this._currentSpeech.done()) {
+        this._currentSpeech._markDone();
       }
-      if (this.stt instanceof STT) {
-        this.stt.off("metrics_collected", this.onMetricsCollected);
+      await this._closeSessionResources();
+      if (this._mainTask) {
+        await this._mainTask.cancelAndWait();
       }
-      if (this.tts instanceof TTS) {
-        this.tts.off("metrics_collected", this.onMetricsCollected);
+      if (this.interruptionDetector) {
+        this.interruptionDetector.off("overlapping_speech", this.onInterruptionOverlappingSpeech);
+        this.interruptionDetector.off("metrics_collected", this.onInterruptionMetricsCollected);
+        this.interruptionDetector.off("error", this.onInterruptionError);
       }
-      if (this.vad instanceof VAD) {
-        this.vad.off("metrics_collected", this.onMetricsCollected);
-      }
-      this.detachAudioInput();
-      await ((_a = this.realtimeSession) == null ? void 0 : _a.close());
-      await ((_b = this.audioRecognition) == null ? void 0 : _b.close());
-      await ((_c = this._mainTask) == null ? void 0 : _c.cancelAndWait());
+      this.agent._agentActivity = void 0;
     } finally {
       unlock();
     }
   }
+  resolveInterruptionDetector() {
+    var _a, _b;
+    const agentInterruptionDetection = (_b = (_a = this.agent.turnHandling) == null ? void 0 : _a.interruption) == null ? void 0 : _b.mode;
+    const sessionInterruptionDetection = this.agentSession.interruptionDetection;
+    if (!(this.stt && this.stt.capabilities.alignedTranscript && this.stt.capabilities.streaming && this.vad && this.turnDetection !== "manual" && this.turnDetection !== "realtime_llm" && !(this.llm instanceof RealtimeModel))) {
+      if (agentInterruptionDetection === "adaptive" || sessionInterruptionDetection === "adaptive") {
+        this.logger.warn(
+          "interruptionDetection is provided, but it's not compatible with the current configuration and will be disabled"
+        );
+      }
+      return void 0;
+    }
+    if (!this.allowInterruptions) {
+      return void 0;
+    }
+    if (agentInterruptionDetection === "vad") {
+      return void 0;
+    }
+    if (sessionInterruptionDetection === "vad") {
+      return void 0;
+    }
+    if (agentInterruptionDetection === void 0 && sessionInterruptionDetection === void 0 && !isHosted() && !isDevMode()) {
+      this.logger.info("adaptive interruption is disabled by default in production mode");
+      return void 0;
+    }
+    try {
+      const detector = new AdaptiveInterruptionDetector();
+      detector.on("overlapping_speech", this.onInterruptionOverlappingSpeech);
+      detector.on("metrics_collected", this.onInterruptionMetricsCollected);
+      detector.on("error", this.onInterruptionError);
+      return detector;
+    } catch (error) {
+      this.logger.warn({ error }, "could not instantiate AdaptiveInterruptionDetector");
+    }
+    return void 0;
+  }
+  restoreInterruptionByAudioActivity() {
+    this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
+  }
+  fallbackToVadInterruption() {
+    if (!this.isInterruptionDetectionEnabled) return;
+    this.isInterruptionDetectionEnabled = false;
+    this.restoreInterruptionByAudioActivity();
+    if (this.interruptionDetector) {
+      this.interruptionDetector.off("overlapping_speech", this.onInterruptionOverlappingSpeech);
+      this.interruptionDetector.off("metrics_collected", this.onInterruptionMetricsCollected);
+      this.interruptionDetector.off("error", this.onInterruptionError);
+      this.interruptionDetector = void 0;
+    }
+    if (this.audioRecognition) {
+      this.audioRecognition.disableInterruptionDetection().catch((err) => {
+        this.logger.warn({ err }, "error while disabling interruption detection");
+      });
+    }
+    this.logger.warn(
+      "adaptive interruption disabled due to unrecoverable error, falling back to VAD-based interruption"
+    );
+  }
+  async _closeSessionResources() {
+    var _a, _b, _c;
+    if (this.llm instanceof LLM) {
+      this.llm.off("metrics_collected", this.onMetricsCollected);
+      this.llm.off("error", this.onModelError);
+    }
+    if (this.realtimeSession) {
+      this.realtimeSession.off("generation_created", this.onRealtimeGenerationCreated);
+      this.realtimeSession.off("input_speech_started", this.onRealtimeInputSpeechStarted);
+      this.realtimeSession.off("input_speech_stopped", this.onRealtimeInputSpeechStopped);
+      this.realtimeSession.off(
+        "input_audio_transcription_completed",
+        this.onRealtimeInputAudioTranscriptionCompleted
+      );
+      this.realtimeSession.off("metrics_collected", this.onMetricsCollected);
+      this.realtimeSession.off("error", this.onModelError);
+    }
+    if (this.stt instanceof STT) {
+      this.stt.off("metrics_collected", this.onMetricsCollected);
+      this.stt.off("error", this.onModelError);
+    }
+    if (this.tts instanceof TTS) {
+      this.tts.off("metrics_collected", this.onMetricsCollected);
+      this.tts.off("error", this.onModelError);
+    }
+    if (this.vad instanceof VAD) {
+      this.vad.off("metrics_collected", this.onMetricsCollected);
+    }
+    this.detachAudioInput();
+    (_a = this.realtimeSpans) == null ? void 0 : _a.clear();
+    await ((_b = this.realtimeSession) == null ? void 0 : _b.close());
+    await ((_c = this.audioRecognition) == null ? void 0 : _c.close());
+    this.realtimeSession = void 0;
+    this.audioRecognition = void 0;
+  }
 }
 function toOaiToolChoice(toolChoice) {
   return toolChoice !== null ? toolChoice : void 0;
 }
 export {
-  AgentActivity
+  AgentActivity,
+  agentActivityStorage,
+  onEnterStorage
 };
 //# sourceMappingURL=agent_activity.js.map