@livekit/agents 1.0.47 → 1.1.0-dev.0

package/src/voice/agent_activity.ts

@@ -7,8 +7,11 @@ import type { Span } from '@opentelemetry/api';
 import { ROOT_CONTEXT, context as otelContext, trace } from '@opentelemetry/api';
 import { Heap } from 'heap-js';
 import { AsyncLocalStorage } from 'node:async_hooks';
-import { ReadableStream } from 'node:stream/web';
-import { type ChatContext, ChatMessage } from '../llm/chat_context.js';
+import { ReadableStream, TransformStream } from 'node:stream/web';
+import type { InterruptionDetectionError } from '../inference/interruption/errors.js';
+import { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js';
+import type { OverlappingSpeechEvent } from '../inference/interruption/types.js';
+import { type ChatContext, ChatMessage, type MetricsReport } from '../llm/chat_context.js';
 import {
   type ChatItem,
   type FunctionCall,
@@ -23,12 +26,14 @@ import {
   type RealtimeSession,
   type ToolChoice,
   type ToolContext,
+  ToolFlag,
 } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import { isSameToolChoice, isSameToolContext } from '../llm/tool_context.js';
 import { log } from '../log.js';
 import type {
   EOUMetrics,
+  InterruptionMetrics,
   LLMMetrics,
   RealtimeModelMetrics,
   STTMetrics,
@@ -56,7 +61,6 @@ import {
   type EndOfTurnInfo,
   type PreemptiveGenerationInfo,
   type RecognitionHooks,
-  type _TurnDetector,
 } from './audio_recognition.js';
 import {
   AgentSessionEventTypes,
@@ -83,6 +87,12 @@ import { SpeechHandle } from './speech_handle.js';
 import { setParticipantSpanAttributes } from './utils.js';
 
 export const agentActivityStorage = new AsyncLocalStorage<AgentActivity>();
+export const onEnterStorage = new AsyncLocalStorage<OnEnterData>();
+
+interface OnEnterData {
+  session: AgentSession;
+  agent: Agent;
+}
 
 interface PreemptiveGeneration {
   speechHandle: SpeechHandle;
@@ -94,6 +104,7 @@ interface PreemptiveGeneration {
   createdAt: number;
 }
 
+// TODO add false interruption handling and barge in handling for https://github.com/livekit/agents/pull/3109/changes
 export class AgentActivity implements RecognitionHooks {
   agent: Agent;
   agentSession: AgentSession;
@@ -104,7 +115,7 @@ export class AgentActivity implements RecognitionHooks {
   private audioRecognition?: AudioRecognition;
   private realtimeSession?: RealtimeSession;
   private realtimeSpans?: Map<string, Span>; // Maps response_id to OTEL span for metrics recording
-  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
+  private turnDetectionMode?: TurnDetectionMode;
   private logger = log();
   private _schedulingPaused = true;
   private _drainBlockedTasks: Task<any>[] = [];
@@ -119,6 +130,43 @@ export class AgentActivity implements RecognitionHooks {
   // default to null as None, which maps to the default provider tool choice value
   private toolChoice: ToolChoice | null = null;
   private _preemptiveGeneration?: PreemptiveGeneration;
+  private interruptionDetector?: AdaptiveInterruptionDetector;
+  private isInterruptionDetectionEnabled: boolean;
+  private isInterruptionByAudioActivityEnabled: boolean;
+  private isDefaultInterruptionByAudioActivityEnabled: boolean;
+
+  private readonly onRealtimeGenerationCreated = (ev: GenerationCreatedEvent): void =>
+    this.onGenerationCreated(ev);
+
+  private readonly onRealtimeInputSpeechStarted = (ev: InputSpeechStartedEvent): void =>
+    this.onInputSpeechStarted(ev);
+
+  private readonly onRealtimeInputSpeechStopped = (ev: InputSpeechStoppedEvent): void =>
+    this.onInputSpeechStopped(ev);
+
+  private readonly onRealtimeInputAudioTranscriptionCompleted = (
+    ev: InputTranscriptionCompleted,
+  ): void => this.onInputAudioTranscriptionCompleted(ev);
+
+  private readonly onModelError = (ev: RealtimeModelError | STTError | TTSError | LLMError): void =>
+    this.onError(ev);
+
+  private readonly onInterruptionOverlappingSpeech = (ev: OverlappingSpeechEvent): void => {
+    this.agentSession.emit(AgentSessionEventTypes.UserOverlappingSpeech, ev);
+  };
+
+  private readonly onInterruptionMetricsCollected = (ev: InterruptionMetrics): void => {
+    this.agentSession.emit(
+      AgentSessionEventTypes.MetricsCollected,
+      createMetricsCollectedEvent({ metrics: ev }),
+    );
+  };
+
+  private readonly onInterruptionError = (ev: InterruptionDetectionError): void => {
+    const errorEvent = createErrorEvent(ev, this.interruptionDetector);
+    this.agentSession.emit(AgentSessionEventTypes.Error, errorEvent);
+    this.agentSession._onError(ev);
+  };
 
   /** @internal */
   _mainTask?: Task<void>;
@@ -126,16 +174,6 @@ export class AgentActivity implements RecognitionHooks {
   _onExitTask?: Task<void>;
   _userTurnCompletedTask?: Task<void>;
 
-  private readonly onRealtimeGenerationCreated = (ev: GenerationCreatedEvent) =>
-    this.onGenerationCreated(ev);
-  private readonly onRealtimeInputSpeechStarted = (ev: InputSpeechStartedEvent) =>
-    this.onInputSpeechStarted(ev);
-  private readonly onRealtimeInputSpeechStopped = (ev: InputSpeechStoppedEvent) =>
-    this.onInputSpeechStopped(ev);
-  private readonly onRealtimeInputAudioTranscriptionCompleted = (ev: InputTranscriptionCompleted) =>
-    this.onInputAudioTranscriptionCompleted(ev);
-  private readonly onModelError = (ev: RealtimeModelError | STTError | TTSError | LLMError) =>
-    this.onError(ev);
   constructor(agent: Agent, agentSession: AgentSession) {
     this.agent = agent;
     this.agentSession = agentSession;
@@ -228,6 +266,16 @@ export class AgentActivity implements RecognitionHooks {
           'for more responsive interruption handling.',
       );
     }
+
+    this.interruptionDetector = this.resolveInterruptionDetector();
+    this.isInterruptionDetectionEnabled = !!this.interruptionDetector;
+
+    // this allows taking over audio interruption temporarily until interruption is detected
+    // by default is is ture unless turnDetection is manual or realtime_llm
+    this.isInterruptionByAudioActivityEnabled =
+      this.turnDetectionMode !== 'manual' && this.turnDetectionMode !== 'realtime_llm';
+
+    this.isDefaultInterruptionByAudioActivityEnabled = this.isInterruptionByAudioActivityEnabled;
   }
 
   async start(): Promise<void> {
@@ -312,6 +360,8 @@ export class AgentActivity implements RecognitionHooks {
       }
     }
 
+    // TODO(parity): Record initial AgentConfigUpdate in chat context
+
     // metrics and error handling
     if (this.llm instanceof LLM) {
       this.llm.on('metrics_collected', this.onMetricsCollected);
@@ -339,8 +389,9 @@ export class AgentActivity implements RecognitionHooks {
       vad: this.vad,
       turnDetector: typeof this.turnDetection === 'string' ? undefined : this.turnDetection,
       turnDetectionMode: this.turnDetectionMode,
-      minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
-      maxEndpointingDelay: this.agentSession.options.maxEndpointingDelay,
+      interruptionDetection: this.interruptionDetector,
+      minEndpointingDelay: this.agentSession.options.turnHandling.endpointing.minDelay,
+      maxEndpointingDelay: this.agentSession.options.turnHandling.endpointing.maxDelay,
       rootSpanContext: this.agentSession.rootSpanContext,
       sttModel: this.stt?.label,
       sttProvider: this.getSttProvider(),
@@ -354,11 +405,13 @@ export class AgentActivity implements RecognitionHooks {
     if (runOnEnter) {
       this._onEnterTask = this.createSpeechTask({
         taskFn: () =>
-          tracer.startActiveSpan(async () => this.agent.onEnter(), {
-            name: 'on_enter',
-            context: trace.setSpan(ROOT_CONTEXT, startSpan),
-            attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
-          }),
+          onEnterStorage.run({ session: this.agentSession, agent: this.agent }, () =>
+            tracer.startActiveSpan(async () => this.agent.onEnter(), {
+              name: 'on_enter',
+              context: trace.setSpan(ROOT_CONTEXT, startSpan),
+              attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+            }),
+          ),
         inlineTask: true,
         name: 'AgentActivity_onEnter',
       });
@@ -412,7 +465,7 @@ export class AgentActivity implements RecognitionHooks {
 
   get allowInterruptions(): boolean {
     // TODO(AJS-51): Allow options to be defined in Agent class
-    return this.agentSession.options.allowInterruptions;
+    return this.agentSession.options.turnHandling.interruption?.mode !== false;
   }
 
   get useTtsAlignedTranscript(): boolean {
@@ -429,6 +482,11 @@ export class AgentActivity implements RecognitionHooks {
     return this.agent.toolCtx;
   }
 
+  /** @internal */
+  get inputStartedAt() {
+    return this.audioRecognition?.inputStartedAt;
+  }
+
   async updateChatCtx(chatCtx: ChatContext): Promise<void> {
     chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });
 
@@ -446,7 +504,27 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
-  updateOptions({ toolChoice }: { toolChoice?: ToolChoice | null }): void {
+  // TODO: Add when AgentConfigUpdate is ported to ChatContext.
+  async updateTools(tools: ToolContext): Promise<void> {
+    this.agent._tools = { ...tools };
+
+    if (this.realtimeSession) {
+      await this.realtimeSession.updateTools(tools);
+    }
+
+    if (this.llm instanceof LLM) {
+      // for realtime LLM, we assume the server will remove unvalid tool messages
+      await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: tools }));
+    }
+  }
+
+  updateOptions({
+    toolChoice,
+    turnDetection,
+  }: {
+    toolChoice?: ToolChoice | null;
+    turnDetection?: TurnDetectionMode;
+  }): void {
     if (toolChoice !== undefined) {
       this.toolChoice = toolChoice;
     }
@@ -454,14 +532,46 @@ export class AgentActivity implements RecognitionHooks {
     if (this.realtimeSession) {
       this.realtimeSession.updateOptions({ toolChoice: this.toolChoice });
     }
+
+    if (turnDetection !== undefined) {
+      this.turnDetectionMode = turnDetection;
+      this.isDefaultInterruptionByAudioActivityEnabled =
+        this.turnDetectionMode !== 'manual' && this.turnDetectionMode !== 'realtime_llm';
+
+      // sync live flag immediately when not speaking so the change takes effect right away
+      if (this.agentSession.agentState !== 'speaking') {
+        this.isInterruptionByAudioActivityEnabled =
+          this.isDefaultInterruptionByAudioActivityEnabled;
+      }
+    }
+
+    if (this.audioRecognition) {
+      this.audioRecognition.updateOptions({ turnDetection: this.turnDetectionMode });
+    }
   }
 
   attachAudioInput(audioStream: ReadableStream<AudioFrame>): void {
     void this.audioStream.close();
     this.audioStream = new MultiInputStream<AudioFrame>();
 
+    // Filter is applied on this.audioStream.stream (downstream of MultiInputStream) rather
+    // than on the source audioStream via pipeThrough. pipeThrough locks its source stream, so
+    // if it were applied directly on audioStream, that lock would survive MultiInputStream.close()
+    // and make audioStream permanently locked for subsequent attachAudioInput calls (e.g. handoff).
+    const aecWarmupAudioFilter = new TransformStream<AudioFrame, AudioFrame>({
+      transform: (frame, controller) => {
+        const shouldDiscardForAecWarmup =
+          this.agentSession.agentState === 'speaking' && this.agentSession._aecWarmupRemaining > 0;
+        if (!shouldDiscardForAecWarmup) {
+          controller.enqueue(frame);
+        }
+      },
+    });
+
     this.audioStreamId = this.audioStream.addInputStream(audioStream);
-    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.tee();
+    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream
+      .pipeThrough(aecWarmupAudioFilter)
+      .tee();
 
     if (this.realtimeSession) {
       this.realtimeSession.setInputAudioStream(realtimeAudioStream);
@@ -614,6 +724,13 @@ export class AgentActivity implements RecognitionHooks {
 
     if (!this.vad) {
       this.agentSession._updateUserState('speaking');
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfOverlapSpeech(
+          0,
+          Date.now(),
+          this.agentSession._userSpeakingSpan,
+        );
+      }
     }
 
     // this.interrupt() is going to raise when allow_interruptions is False,
@@ -632,6 +749,9 @@ export class AgentActivity implements RecognitionHooks {
     this.logger.info(ev, 'onInputSpeechStopped');
 
     if (!this.vad) {
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onEndOfOverlapSpeech(Date.now(), this.agentSession._userSpeakingSpan);
+      }
       this.agentSession._updateUserState('listening');
     }
 
@@ -705,15 +825,32 @@ export class AgentActivity implements RecognitionHooks {
   onStartOfSpeech(ev: VADEvent): void {
     let speechStartTime = Date.now();
     if (ev) {
-      speechStartTime = speechStartTime - ev.speechDuration;
+      // Subtract both speechDuration and inferenceDuration to correct for VAD model latency.
+      speechStartTime = speechStartTime - ev.speechDuration - ev.inferenceDuration;
     }
     this.agentSession._updateUserState('speaking', speechStartTime);
+    if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+      // Pass speechStartTime as the absolute startedAt timestamp.
+      this.audioRecognition.onStartOfOverlapSpeech(
+        ev.speechDuration,
+        speechStartTime,
+        this.agentSession._userSpeakingSpan,
+      );
+    }
   }
 
   onEndOfSpeech(ev: VADEvent): void {
     let speechEndTime = Date.now();
     if (ev) {
-      speechEndTime = speechEndTime - ev.silenceDuration;
+      // Subtract both silenceDuration and inferenceDuration to correct for VAD model latency.
+      speechEndTime = speechEndTime - ev.silenceDuration - ev.inferenceDuration;
+    }
+    if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+      // Pass speechEndTime as the absolute endedAt timestamp.
+      this.audioRecognition.onEndOfOverlapSpeech(
+        speechEndTime,
+        this.agentSession._userSpeakingSpan,
+      );
     }
     this.agentSession._updateUserState('listening', speechEndTime);
   }
@@ -724,12 +861,21 @@ export class AgentActivity implements RecognitionHooks {
       return;
     }
 
-    if (ev.speechDuration >= this.agentSession.options.minInterruptionDuration) {
+    if (ev.speechDuration >= this.agentSession.options.turnHandling.interruption?.minDuration) {
       this.interruptByAudioActivity();
     }
   }
 
   private interruptByAudioActivity(): void {
+    if (!this.isInterruptionByAudioActivityEnabled) {
+      return;
+    }
+
+    if (this.agentSession._aecWarmupRemaining > 0) {
+      // Disable interruption from audio activity while AEC warmup is active.
+      return;
+    }
+
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
       // skip speech handle interruption if server side turn detection is enabled
       return;
@@ -739,7 +885,11 @@ export class AgentActivity implements RecognitionHooks {
     // - Always apply minInterruptionWords filtering when STT is available and minInterruptionWords > 0
     // - Apply check to all STT results: empty string, undefined, or any length
     // - This ensures consistent behavior across all interruption scenarios
-    if (this.stt && this.agentSession.options.minInterruptionWords > 0 && this.audioRecognition) {
+    if (
+      this.stt &&
+      this.agentSession.options.turnHandling.interruption?.minWords > 0 &&
+      this.audioRecognition
+    ) {
       const text = this.audioRecognition.currentTranscript;
       // TODO(shubhra): better word splitting for multi-language
 
@@ -749,7 +899,7 @@ export class AgentActivity implements RecognitionHooks {
 
       // Only allow interruption if word count meets or exceeds minInterruptionWords
       // This applies to all cases: empty strings, partial speech, and full speech
-      if (wordCount < this.agentSession.options.minInterruptionWords) {
+      if (wordCount < this.agentSession.options.turnHandling.interruption?.minWords) {
         return;
       }
     }
@@ -770,6 +920,14 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
+  onInterruption(ev: OverlappingSpeechEvent) {
+    this.restoreInterruptionByAudioActivity();
+    this.interruptByAudioActivity();
+    if (this.audioRecognition) {
+      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.timestamp);
+    }
+  }
+
   onInterimTranscript(ev: SpeechEvent): void {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
       // skip stt transcription if userTranscription is enabled on the realtime model
@@ -845,6 +1003,7 @@ export class AgentActivity implements RecognitionHooks {
     const userMessage = ChatMessage.create({
       role: 'user',
       content: info.newTranscript,
+      transcriptConfidence: info.transcriptConfidence,
     });
     const chatCtx = this.agent.chatCtx.copy();
     const speechHandle = this.generateReply({
@@ -940,16 +1099,16 @@ export class AgentActivity implements RecognitionHooks {
       this._currentSpeech &&
       this._currentSpeech.allowInterruptions &&
       !this._currentSpeech.interrupted &&
-      this.agentSession.options.minInterruptionWords > 0
+      this.agentSession.options.turnHandling.interruption?.minWords > 0
     ) {
       const wordCount = splitWords(info.newTranscript, true).length;
-      if (wordCount < this.agentSession.options.minInterruptionWords) {
+      if (wordCount < this.agentSession.options.turnHandling.interruption?.minWords) {
         // avoid interruption if the new_transcript contains fewer words than minInterruptionWords
         this.cancelPreemptiveGeneration();
         this.logger.info(
           {
             wordCount,
-            minInterruptionWords: this.agentSession.options.minInterruptionWords,
+            minInterruptionWords: this.agentSession.options.turnHandling.interruption.minWords,
           },
           'skipping user input, word count below minimum interruption threshold',
         );
@@ -1129,12 +1288,25 @@ export class AgentActivity implements RecognitionHooks {
         instructions = `${this.agent.instructions}\n${instructions}`;
       }
 
+      // Filter out tools with IGNORE_ON_ENTER flag when generateReply is called inside onEnter
+      const onEnterData = onEnterStorage.getStore();
+      const shouldFilterTools =
+        onEnterData?.agent === this.agent && onEnterData?.session === this.agentSession;
+
+      const tools = shouldFilterTools
+        ? Object.fromEntries(
+            Object.entries(this.agent.toolCtx).filter(
+              ([, fnTool]) => !(fnTool.flags & ToolFlag.IGNORE_ON_ENTER),
+            ),
+          )
+        : this.agent.toolCtx;
+
       const task = this.createSpeechTask({
         taskFn: (abortController: AbortController) =>
           this.pipelineReplyTask(
             handle,
             chatCtx ?? this.agent.chatCtx,
-            this.agent.toolCtx,
+            tools,
             {
               toolChoice: toOaiToolChoice(toolChoice !== undefined ? toolChoice : this.toolChoice),
             },
@@ -1234,6 +1406,7 @@ export class AgentActivity implements RecognitionHooks {
     let userMessage: ChatMessage | undefined = ChatMessage.create({
       role: 'user',
       content: info.newTranscript,
+      transcriptConfidence: info.transcriptConfidence,
     });
 
     // create a temporary mutable chat context to pass to onUserTurnCompleted
@@ -1260,6 +1433,24 @@ export class AgentActivity implements RecognitionHooks {
       return;
     }
 
+    const userMetricsReport: MetricsReport = {};
+    if (info.startedSpeakingAt !== undefined) {
+      userMetricsReport.startedSpeakingAt = info.startedSpeakingAt / 1000; // ms -> seconds
+    }
+    if (info.stoppedSpeakingAt !== undefined) {
+      userMetricsReport.stoppedSpeakingAt = info.stoppedSpeakingAt / 1000; // ms -> seconds
+    }
+    if (info.transcriptionDelay !== undefined) {
+      userMetricsReport.transcriptionDelay = info.transcriptionDelay / 1000; // ms -> seconds
+    }
+    if (info.endOfUtteranceDelay !== undefined) {
+      userMetricsReport.endOfTurnDelay = info.endOfUtteranceDelay / 1000; // ms -> seconds
+    }
+    userMetricsReport.onUserTurnCompletedDelay = callbackDuration / 1000; // ms -> seconds
+    if (userMessage) {
+      userMessage.metrics = userMetricsReport;
+    }
+
     let speechHandle: SpeechHandle | undefined;
     if (this._preemptiveGeneration !== undefined) {
       const preemptive = this._preemptiveGeneration;
@@ -1272,6 +1463,14 @@ export class AgentActivity implements RecognitionHooks {
         isSameToolChoice(preemptive.toolChoice, this.toolChoice)
       ) {
         speechHandle = preemptive.speechHandle;
+        // The preemptive userMessage was created without metrics.
+        // Copy the metrics and transcriptConfidence from the new userMessage
+        // to the preemptive message BEFORE scheduling (so the pipeline inserts
+        // the message with metrics already set).
+        if (preemptive.userMessage && userMessage) {
+          preemptive.userMessage.metrics = userMetricsReport;
+          preemptive.userMessage.transcriptConfidence = userMessage.transcriptConfidence;
+        }
         this.scheduleSpeech(speechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
         this.logger.debug(
           {
@@ -1365,11 +1564,19 @@ export class AgentActivity implements RecognitionHooks {
       tasks.push(textForwardTask);
     }
 
+    let replyStartedSpeakingAt: number | undefined;
+    let replyTtsGenData: _TTSGenerationData | null = null;
+
     const onFirstFrame = (startedSpeakingAt?: number) => {
+      replyStartedSpeakingAt = startedSpeakingAt ?? Date.now();
       this.agentSession._updateAgentState('speaking', {
         startTime: startedSpeakingAt,
         otelContext: speechHandle._agentTurnContext,
       });
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfAgentSpeech();
+        this.isInterruptionByAudioActivityEnabled = false;
+      }
     };
 
     if (!audioOutput) {
@@ -1387,8 +1594,11 @@ export class AgentActivity implements RecognitionHooks {
           audioSource,
           modelSettings,
           replyAbortController,
+          this.tts?.model,
+          this.tts?.provider,
         );
         tasks.push(ttsTask);
+        replyTtsGenData = ttsGenData;
 
         const [forwardTask, _audioOut] = performAudioForwarding(
           ttsGenData.audioStream,
@@ -1428,10 +1638,21 @@ export class AgentActivity implements RecognitionHooks {
     }
 
     if (addToChatCtx) {
+      const replyStoppedSpeakingAt = Date.now();
+      const replyAssistantMetrics: MetricsReport = {};
+      if (replyTtsGenData?.ttfb !== undefined) {
+        replyAssistantMetrics.ttsNodeTtfb = replyTtsGenData.ttfb;
+      }
+      if (replyStartedSpeakingAt !== undefined) {
+        replyAssistantMetrics.startedSpeakingAt = replyStartedSpeakingAt / 1000; // ms -> seconds
+        replyAssistantMetrics.stoppedSpeakingAt = replyStoppedSpeakingAt / 1000; // ms -> seconds
+      }
+
       const message = ChatMessage.create({
         role: 'assistant',
         content: textOut?.text || '',
         interrupted: speechHandle.interrupted,
+        metrics: replyAssistantMetrics,
       });
       this.agent._chatCtx.insert(message);
       this.agentSession._conversationItemAdded(message);
@@ -1439,6 +1660,10 @@ export class AgentActivity implements RecognitionHooks {
 
     if (this.agentSession.agentState === 'speaking') {
       this.agentSession._updateAgentState('listening');
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onEndOfAgentSpeech(Date.now());
+      }
+      this.restoreInterruptionByAudioActivity();
     }
   }
 
@@ -1452,6 +1677,7 @@ export class AgentActivity implements RecognitionHooks {
     newMessage,
     toolsMessages,
     span,
+    _previousUserMetrics,
   }: {
     speechHandle: SpeechHandle;
     chatCtx: ChatContext;
@@ -1462,6 +1688,7 @@ export class AgentActivity implements RecognitionHooks {
     newMessage?: ChatMessage;
     toolsMessages?: ChatItem[];
     span: Span;
+    _previousUserMetrics?: MetricsReport;
   }): Promise<void> => {
     speechHandle._agentTurnContext = otelContext.active();
 
@@ -1514,6 +1741,8 @@ export class AgentActivity implements RecognitionHooks {
       toolCtx,
       modelSettings,
       replyAbortController,
+      this.llm?.model,
+      this.llm?.provider,
     );
     tasks.push(llmTask);
 
@@ -1530,6 +1759,8 @@ export class AgentActivity implements RecognitionHooks {
         ttsTextInput,
         modelSettings,
         replyAbortController,
+        this.tts?.model,
+        this.tts?.provider,
       );
       tasks.push(ttsTask);
     } else {
@@ -1539,10 +1770,12 @@ export class AgentActivity implements RecognitionHooks {
 
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForScheduled()]);
 
+    let userMetrics: MetricsReport | undefined = _previousUserMetrics;
     // Add new message to actual chat context if the speech is scheduled
     if (newMessage && speechHandle.scheduled) {
       this.agent._chatCtx.insert(newMessage);
       this.agentSession._conversationItemAdded(newMessage);
+      userMetrics = newMessage.metrics;
     }
 
     if (speechHandle.interrupted) {
@@ -1588,11 +1821,17 @@ export class AgentActivity implements RecognitionHooks {
       textOut = _textOut;
     }
 
+    let agentStartedSpeakingAt: number | undefined;
     const onFirstFrame = (startedSpeakingAt?: number) => {
+      agentStartedSpeakingAt = startedSpeakingAt ?? Date.now();
       this.agentSession._updateAgentState('speaking', {
         startTime: startedSpeakingAt,
         otelContext: speechHandle._agentTurnContext,
       });
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        this.audioRecognition.onStartOfAgentSpeech();
+        this.isInterruptionByAudioActivityEnabled = false;
+      }
     };
 
     let audioOut: _AudioOut | null = null;
@@ -1649,6 +1888,29 @@ export class AgentActivity implements RecognitionHooks {
       await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]);
     }
 
+    const agentStoppedSpeakingAt = Date.now();
+    const assistantMetrics: MetricsReport = {};
+
+    if (llmGenData.ttft !== undefined) {
+      assistantMetrics.llmNodeTtft = llmGenData.ttft; // already in seconds
+    }
+    if (ttsGenData?.ttfb !== undefined) {
+      assistantMetrics.ttsNodeTtfb = ttsGenData.ttfb; // already in seconds
+    }
+    if (agentStartedSpeakingAt !== undefined) {
+      assistantMetrics.startedSpeakingAt = agentStartedSpeakingAt / 1000; // ms -> seconds
+      assistantMetrics.stoppedSpeakingAt = agentStoppedSpeakingAt / 1000; // ms -> seconds
+
+      if (userMetrics?.stoppedSpeakingAt !== undefined) {
+        const e2eLatency = agentStartedSpeakingAt / 1000 - userMetrics.stoppedSpeakingAt;
+        assistantMetrics.e2eLatency = e2eLatency;
+        span.setAttribute(traceTypes.ATTR_E2E_LATENCY, e2eLatency);
+      }
+    }
+
+    span.setAttribute(traceTypes.ATTR_SPEECH_INTERRUPTED, speechHandle.interrupted);
+    let hasSpeechMessage = false;
+
     // add the tools messages that triggers this reply to the chat context
     if (toolsMessages) {
       for (const msg of toolsMessages) {
@@ -1703,45 +1965,54 @@ export class AgentActivity implements RecognitionHooks {
       }
 
       if (forwardedText) {
+        hasSpeechMessage = true;
         const message = ChatMessage.create({
           role: 'assistant',
           content: forwardedText,
           id: llmGenData.id,
           interrupted: true,
           createdAt: replyStartedAt,
+          metrics: assistantMetrics,
         });
         chatCtx.insert(message);
         this.agent._chatCtx.insert(message);
         speechHandle._itemAdded([message]);
         this.agentSession._conversationItemAdded(message);
+        span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, forwardedText);
       }
 
       if (this.agentSession.agentState === 'speaking') {
         this.agentSession._updateAgentState('listening');
+        if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+          this.restoreInterruptionByAudioActivity();
+        }
       }
 
       this.logger.info(
         { speech_id: speechHandle.id, message: forwardedText },
         'playout completed with interrupt',
       );
-      // TODO(shubhra) add chat message to speech handle
       speechHandle._markGenerationDone();
       await executeToolsTask.cancelAndWait(AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
       return;
     }
 
     if (textOut && textOut.text) {
+      hasSpeechMessage = true;
       const message = ChatMessage.create({
         role: 'assistant',
         id: llmGenData.id,
         interrupted: false,
         createdAt: replyStartedAt,
         content: textOut.text,
+        metrics: assistantMetrics,
       });
       chatCtx.insert(message);
       this.agent._chatCtx.insert(message);
       speechHandle._itemAdded([message]);
       this.agentSession._conversationItemAdded(message);
+      span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, textOut.text);
       this.logger.info(
         { speech_id: speechHandle.id, message: textOut.text },
         'playout completed without interruption',
@@ -1752,6 +2023,12 @@ export class AgentActivity implements RecognitionHooks {
       this.agentSession._updateAgentState('thinking');
     } else if (this.agentSession.agentState === 'speaking') {
       this.agentSession._updateAgentState('listening');
+      if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+          this.restoreInterruptionByAudioActivity();
+        }
+      }
     }
 
     // mark the playout done before waiting for the tool execution
@@ -1811,6 +2088,7 @@ export class AgentActivity implements RecognitionHooks {
             instructions,
             undefined,
             toolMessages,
+            hasSpeechMessage ? undefined : userMetrics,
           ),
         ownedSpeechHandle: speechHandle,
         name: 'AgentActivity.pipelineReply',
@@ -1844,6 +2122,7 @@ export class AgentActivity implements RecognitionHooks {
     instructions?: string,
     newMessage?: ChatMessage,
     toolsMessages?: ChatItem[],
+    _previousUserMetrics?: MetricsReport,
   ): Promise<void> =>
     tracer.startActiveSpan(
       async (span) =>
@@ -1857,6 +2136,7 @@ export class AgentActivity implements RecognitionHooks {
           newMessage,
           toolsMessages,
           span,
+          _previousUserMetrics,
         }),
       {
         name: 'agent_turn',
@@ -2007,6 +2287,8 @@ export class AgentActivity implements RecognitionHooks {
                 ttsTextInput,
                 modelSettings,
                 abortController,
+                this.tts?.model,
+                this.tts?.provider,
               );
               tasks.push(ttsTask);
               realtimeAudioResult = ttsGenData.audioStream;
@@ -2516,6 +2798,14 @@ export class AgentActivity implements RecognitionHooks {
       if (this._mainTask) {
         await this._mainTask.cancelAndWait();
       }
+      if (this.interruptionDetector) {
+        this.interruptionDetector.off(
+          'user_overlapping_speech',
+          this.onInterruptionOverlappingSpeech,
+        );
+        this.interruptionDetector.off('metrics_collected', this.onInterruptionMetricsCollected);
+        this.interruptionDetector.off('error', this.onInterruptionError);
+      }
 
       this.agent._agentActivity = undefined;
     } finally {
@@ -2523,6 +2813,53 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
+  private resolveInterruptionDetector(): AdaptiveInterruptionDetector | undefined {
+    const interruptionDetection =
+      this.agent.interruptionDetection ?? this.agentSession.interruptionDetection;
+    if (
+      !(
+        this.stt &&
+        this.stt.capabilities.alignedTranscript &&
+        this.stt.capabilities.streaming &&
+        this.vad &&
+        this.turnDetection !== 'manual' &&
+        this.turnDetection !== 'realtime_llm' &&
+        !(this.llm instanceof RealtimeModel)
+      )
+    ) {
+      if (interruptionDetection === 'adaptive') {
+        this.logger.warn(
+          "interruptionDetection is provided, but it's not compatible with the current configuration and will be disabled",
+        );
+        return undefined;
+      }
+    }
+
+    if (
+      (interruptionDetection !== undefined && interruptionDetection === false) ||
+      interruptionDetection === 'vad'
+    ) {
+      return undefined;
+    }
+
+    try {
+      const detector = new AdaptiveInterruptionDetector();
+
+      detector.on('user_overlapping_speech', this.onInterruptionOverlappingSpeech);
+      detector.on('metrics_collected', this.onInterruptionMetricsCollected);
+      detector.on('error', this.onInterruptionError);
+
+      return detector;
+    } catch (error: unknown) {
+      this.logger.warn({ error }, 'could not instantiate AdaptiveInterruptionDetector');
+    }
+    return undefined;
+  }
+
+  private restoreInterruptionByAudioActivity(): void {
+    this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
+  }
+
   private async _closeSessionResources(): Promise<void> {
     // Unregister event handlers to prevent duplicate metrics
     if (this.llm instanceof LLM) {