@livekit/agents 1.1.0 → 1.2.0

package/src/voice/audio_recognition.ts

@@ -1,73 +1,150 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { ParticipantKind } from '@livekit/rtc-node';
 import { AudioFrame } from '@livekit/rtc-node';
+import {
+  type Context,
+  ROOT_CONTEXT,
+  type Span,
+  context as otelContext,
+  trace,
+} from '@opentelemetry/api';
 import type { WritableStreamDefaultWriter } from 'node:stream/web';
 import { ReadableStream } from 'node:stream/web';
+import { isAPIError } from '../_exceptions.js';
+import { apiConnectDefaults, intervalForRetry } from '../inference/interruption/defaults.js';
+import { InterruptionDetectionError } from '../inference/interruption/errors.js';
+import type { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js';
+import { InterruptionStreamSentinel } from '../inference/interruption/interruption_stream.js';
+import {
+  type InterruptionSentinel,
+  type OverlappingSpeechEvent,
+} from '../inference/interruption/types.js';
+import type { LanguageCode } from '../language.js';
 import { type ChatContext } from '../llm/chat_context.js';
 import { log } from '../log.js';
 import { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/deferred_stream.js';
 import { IdentityTransform } from '../stream/identity_transform.js';
 import { mergeReadableStreams } from '../stream/merge_readable_streams.js';
+import { type StreamChannel, createStreamChannel } from '../stream/stream_channel.js';
 import { type SpeechEvent, SpeechEventType } from '../stt/stt.js';
-import { Task, delay } from '../utils.js';
+import { traceTypes, tracer } from '../telemetry/index.js';
+import { Task, delay, waitForAbort } from '../utils.js';
 import { type VAD, type VADEvent, VADEventType } from '../vad.js';
 import type { TurnDetectionMode } from './agent_session.js';
 import type { STTNode } from './io.js';
+import { setParticipantSpanAttributes } from './utils.js';
 
 export interface EndOfTurnInfo {
+  /** The new transcript text from the user's speech. */
   newTranscript: string;
+  /** Confidence score of the transcript (0-1). */
+  transcriptConfidence: number;
+  /** Delay from speech stop to final transcription in milliseconds. */
   transcriptionDelay: number;
+  /** Delay from speech stop to end of utterance detection in milliseconds. */
   endOfUtteranceDelay: number;
+  /** Timestamp when user started speaking (milliseconds since epoch). */
+  startedSpeakingAt: number | undefined;
+  /** Timestamp when user stopped speaking (milliseconds since epoch). */
+  stoppedSpeakingAt: number | undefined;
+}
+
+export interface PreemptiveGenerationInfo {
+  newTranscript: string;
+  transcriptConfidence: number;
 }
 
 export interface RecognitionHooks {
+  onInterruption: (ev: OverlappingSpeechEvent) => void;
   onStartOfSpeech: (ev: VADEvent) => void;
   onVADInferenceDone: (ev: VADEvent) => void;
   onEndOfSpeech: (ev: VADEvent) => void;
   onInterimTranscript: (ev: SpeechEvent) => void;
   onFinalTranscript: (ev: SpeechEvent) => void;
   onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;
+  onPreemptiveGeneration: (info: PreemptiveGenerationInfo) => void;
 
   retrieveChatCtx: () => ChatContext;
 }
 
 export interface _TurnDetector {
-  unlikelyThreshold: (language?: string) => Promise<number | undefined>;
-  supportsLanguage: (language?: string) => Promise<boolean>;
-  predictEndOfTurn(chatCtx: ChatContext): Promise<number>;
+  /** The model name used by this turn detector. */
+  readonly model: string;
+  /** The provider name for this turn detector. */
+  readonly provider: string;
+  unlikelyThreshold: (language?: LanguageCode) => Promise<number | undefined>;
+  supportsLanguage: (language?: LanguageCode) => Promise<boolean>;
+  predictEndOfTurn(chatCtx: ChatContext, timeout?: number): Promise<number>;
 }
 
 export interface AudioRecognitionOptions {
+  /** Hooks for recognition events. */
   recognitionHooks: RecognitionHooks;
+  /** Speech-to-text node. */
   stt?: STTNode;
+  /** Voice activity detection. */
   vad?: VAD;
+  /** Turn detector for end-of-turn prediction. */
   turnDetector?: _TurnDetector;
-  turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
+  /** Turn detection mode. */
+  turnDetectionMode?: TurnDetectionMode;
+  interruptionDetection?: AdaptiveInterruptionDetector;
+  /** Minimum endpointing delay in milliseconds. */
   minEndpointingDelay: number;
+  /** Maximum endpointing delay in milliseconds. */
   maxEndpointingDelay: number;
+  /** Root span context for tracing. */
+  rootSpanContext?: Context;
+  /** STT model name for tracing */
+  sttModel?: string;
+  /** STT provider name for tracing */
+  sttProvider?: string;
+  /** Getter for linked participant for span attribution */
+  getLinkedParticipant?: () => ParticipantLike | undefined;
+}
+
+/**
+ * Minimal participant shape for span attribution.
+ * Compatible with both `LocalParticipant` and `RemoteParticipant` from `@livekit/rtc-node`.
+ */
+export interface ParticipantLike {
+  sid: string | undefined;
+  identity: string;
+  kind: ParticipantKind;
 }
 
+// TODO add ability to update stt/vad/interruption-detection
 export class AudioRecognition {
   private hooks: RecognitionHooks;
   private stt?: STTNode;
   private vad?: VAD;
   private turnDetector?: _TurnDetector;
-  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
+  private turnDetectionMode?: TurnDetectionMode;
   private minEndpointingDelay: number;
   private maxEndpointingDelay: number;
-  private lastLanguage?: string;
+  private lastLanguage?: LanguageCode;
+  private rootSpanContext?: Context;
+  private sttModel?: string;
+  private sttProvider?: string;
+  private getLinkedParticipant?: () => ParticipantLike | undefined;
 
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
   private logger = log();
   private lastFinalTranscriptTime = 0;
   private audioTranscript = '';
   private audioInterimTranscript = '';
-  private lastSpeakingTime = 0;
+  private audioPreflightTranscript = '';
+  private finalTranscriptConfidence: number[] = [];
+  private lastSpeakingTime: number | undefined;
+  private speechStartTime: number | undefined;
   private userTurnCommitted = false;
   private speaking = false;
   private sampleRate?: number;
 
+  private userTurnSpan?: Span;
+
   private vadInputStream: ReadableStream<AudioFrame>;
   private sttInputStream: ReadableStream<AudioFrame>;
   private silenceAudioTransform = new IdentityTransform<AudioFrame>();
@@ -78,6 +155,16 @@ export class AudioRecognition {
   private commitUserTurnTask?: Task<void>;
   private vadTask?: Task<void>;
   private sttTask?: Task<void>;
+  private interruptionTask?: Task<void>;
+
+  // interruption detection
+  private interruptionDetection?: AdaptiveInterruptionDetector;
+  private _inputStartedAt?: number;
+  private ignoreUserTranscriptUntil?: number;
+  private transcriptBuffer: SpeechEvent[];
+  private isInterruptionEnabled: boolean;
+  private isAgentSpeaking: boolean;
+  private interruptionStreamChannel?: StreamChannel<InterruptionSentinel | AudioFrame>;
 
   constructor(opts: AudioRecognitionOptions) {
     this.hooks = opts.recognitionHooks;
@@ -88,11 +175,35 @@ export class AudioRecognition {
     this.minEndpointingDelay = opts.minEndpointingDelay;
     this.maxEndpointingDelay = opts.maxEndpointingDelay;
     this.lastLanguage = undefined;
+    this.rootSpanContext = opts.rootSpanContext;
+    this.sttModel = opts.sttModel;
+    this.sttProvider = opts.sttProvider;
+    this.getLinkedParticipant = opts.getLinkedParticipant;
 
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
-    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
-    this.vadInputStream = vadInputStream;
-    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);
+    this.interruptionDetection = opts.interruptionDetection;
+    this.transcriptBuffer = [];
+    this.isInterruptionEnabled = !!(opts.interruptionDetection && opts.vad);
+    this.isAgentSpeaking = false;
+
+    if (opts.interruptionDetection) {
+      const [vadInputStream, teedInput] = this.deferredInputStream.stream.tee();
+      const [inputStream, sttInputStream] = teedInput.tee();
+      this.vadInputStream = vadInputStream;
+      this.sttInputStream = mergeReadableStreams(
+        sttInputStream,
+        this.silenceAudioTransform.readable,
+      );
+      this.interruptionStreamChannel = createStreamChannel();
+      this.interruptionStreamChannel.addStreamInput(inputStream);
+    } else {
+      const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
+      this.vadInputStream = vadInputStream;
+      this.sttInputStream = mergeReadableStreams(
+        sttInputStream,
+        this.silenceAudioTransform.readable,
+      );
+    }
     this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();
   }
 
@@ -106,6 +217,16 @@ export class AudioRecognition {
     return this.audioTranscript;
   }
 
+  /** @internal */
+  get inputStartedAt() {
+    return this._inputStartedAt;
+  }
+
+  /** @internal */
+  updateOptions(options: { turnDetection: TurnDetectionMode | undefined }): void {
+    this.turnDetectionMode = options.turnDetection;
+  }
+
   async start() {
     this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));
     this.vadTask.result.catch((err) => {
@@ -116,6 +237,251 @@ export class AudioRecognition {
     this.sttTask.result.catch((err) => {
       this.logger.error(`Error running STT task: ${err}`);
     });
+
+    this.interruptionTask = Task.from(({ signal }) =>
+      this.createInterruptionTask(this.interruptionDetection, signal),
+    );
+    this.interruptionTask.result.catch((err) => {
+      this.logger.error(`Error running interruption task: ${err}`);
+    });
+  }
+
+  async stop() {
+    await this.sttTask?.cancelAndWait();
+    await this.vadTask?.cancelAndWait();
+    await this.interruptionTask?.cancelAndWait();
+  }
+
+  async disableInterruptionDetection(): Promise<void> {
+    this.isInterruptionEnabled = false;
+    this.interruptionDetection = undefined;
+    await this.interruptionTask?.cancelAndWait();
+    this.interruptionTask = undefined;
+    await this.interruptionStreamChannel?.close();
+    this.interruptionStreamChannel = undefined;
+  }
+
+  async onStartOfAgentSpeech() {
+    this.isAgentSpeaking = true;
+    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.agentSpeechStarted());
+  }
+
+  async onEndOfAgentSpeech(ignoreUserTranscriptUntil: number) {
+    if (!this.isInterruptionEnabled) {
+      this.isAgentSpeaking = false;
+      return;
+    }
+
+    const inputOpen = await this.trySendInterruptionSentinel(
+      InterruptionStreamSentinel.agentSpeechEnded(),
+    );
+    if (!inputOpen) {
+      this.isAgentSpeaking = false;
+      return;
+    }
+
+    if (this.isAgentSpeaking) {
+      if (this.ignoreUserTranscriptUntil === undefined) {
+        this.onEndOfOverlapSpeech(Date.now());
+      }
+      this.ignoreUserTranscriptUntil = this.ignoreUserTranscriptUntil
+        ? Math.min(ignoreUserTranscriptUntil, this.ignoreUserTranscriptUntil)
+        : ignoreUserTranscriptUntil;
+
+      // flush held transcripts if possible
+      await this.flushHeldTranscripts();
+    }
+    this.isAgentSpeaking = false;
+  }
+
+  /** Start interruption inference when agent is speaking and overlap speech starts. */
+  async onStartOfOverlapSpeech(speechDuration: number, startedAt: number, userSpeakingSpan?: Span) {
+    if (this.isAgentSpeaking) {
+      this.trySendInterruptionSentinel(
+        InterruptionStreamSentinel.overlapSpeechStarted(
+          speechDuration,
+          startedAt,
+          userSpeakingSpan,
+        ),
+      );
+    }
+  }
+
+  /** End interruption inference when overlap speech ends. */
+  async onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span) {
+    if (!this.isInterruptionEnabled) {
+      return;
+    }
+    if (userSpeakingSpan && userSpeakingSpan.isRecording()) {
+      userSpeakingSpan.setAttribute(traceTypes.ATTR_IS_INTERRUPTION, 'false');
+    }
+
+    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.overlapSpeechEnded(endedAt));
+  }
+
+  /**
+   * Flush held transcripts whose *end time* is after the ignoreUserTranscriptUntil timestamp.
+   * If the event has no timestamps, we assume it is the same as the next valid event.
+   */
+  private async flushHeldTranscripts() {
+    if (
+      !this.isInterruptionEnabled ||
+      this.ignoreUserTranscriptUntil === undefined ||
+      this.transcriptBuffer.length === 0
+    ) {
+      return;
+    }
+
+    if (!this._inputStartedAt) {
+      this.transcriptBuffer = [];
+      this.ignoreUserTranscriptUntil = undefined;
+      return;
+    }
+
+    let emitFromIndex: number | null = null;
+    let shouldFlush = false;
+
+    for (let i = 0; i < this.transcriptBuffer.length; i++) {
+      const ev = this.transcriptBuffer[i];
+      if (!ev || !ev.alternatives || ev.alternatives.length === 0) {
+        emitFromIndex = Math.min(emitFromIndex ?? i, i);
+        continue;
+      }
+      const firstAlternative = ev.alternatives[0];
+      if (
+        firstAlternative.startTime === firstAlternative.endTime &&
+        firstAlternative.startTime === 0
+      ) {
+        this.transcriptBuffer = [];
+        this.ignoreUserTranscriptUntil = undefined;
+        return;
+      }
+
+      if (this.#alternativeEndsBeforeIgnoreWindow(firstAlternative)) {
+        emitFromIndex = null;
+      } else {
+        emitFromIndex = Math.min(emitFromIndex ?? i, i);
+        shouldFlush = true;
+        break;
+      }
+    }
+
+    const eventsToEmit =
+      emitFromIndex !== null && shouldFlush ? this.transcriptBuffer.slice(emitFromIndex) : [];
+
+    this.transcriptBuffer = [];
+    this.ignoreUserTranscriptUntil = undefined;
+
+    for (const event of eventsToEmit) {
+      this.logger.trace(
+        {
+          event: event.type,
+        },
+        're-emitting held user transcript',
+      );
+      this.onSTTEvent(event);
+    }
+  }
+
+  #alternativeEndsBeforeIgnoreWindow(
+    alternative: NonNullable<SpeechEvent['alternatives']>[number],
+  ): boolean {
+    if (
+      this.ignoreUserTranscriptUntil === undefined ||
+      !this._inputStartedAt ||
+      alternative.startTime <= 0
+    ) {
+      return false;
+    }
+
+    // `SpeechData.startTime` is in seconds relative to audio start, while `inputStartedAt` and
+    // `ignoreUserTranscriptUntil` are epoch milliseconds.
+    return alternative.startTime * 1000 + this._inputStartedAt < this.ignoreUserTranscriptUntil;
+  }
+
+  private shouldHoldSttEvent(ev: SpeechEvent): boolean {
+    if (!this.isInterruptionEnabled) {
+      return false;
+    }
+    if (this.isAgentSpeaking) {
+      return true;
+    }
+
+    // reset when the user starts speaking after the agent speech
+    if (ev.type === SpeechEventType.START_OF_SPEECH) {
+      this.ignoreUserTranscriptUntil = undefined;
+      this.transcriptBuffer = [];
+      return false;
+    }
+
+    if (this.ignoreUserTranscriptUntil === undefined) {
+      return false;
+    }
+    // sentinel events are always held until we have something concrete to release them
+    if (!ev.alternatives || ev.alternatives.length === 0) {
+      return true;
+    }
+
+    const alternative = ev.alternatives[0];
+
+    if (
+      alternative.startTime !== alternative.endTime &&
+      this.#alternativeEndsBeforeIgnoreWindow(alternative)
+    ) {
+      return true;
+    }
+    return false;
+  }
+
+  private async trySendInterruptionSentinel(
+    frame: AudioFrame | InterruptionSentinel,
+  ): Promise<boolean> {
+    if (
+      this.isInterruptionEnabled &&
+      this.interruptionStreamChannel &&
+      !this.interruptionStreamChannel.closed
+    ) {
+      try {
+        await this.interruptionStreamChannel.write(frame);
+        return true;
+      } catch (e: unknown) {
+        this.logger.warn(
+          `could not forward interruption sentinel: ${e instanceof Error ? e.message : String(e)}`,
+        );
+      }
+    }
+    return false;
+  }
+
+  private ensureUserTurnSpan(startTime?: number): Span {
+    if (this.userTurnSpan && this.userTurnSpan.isRecording()) {
+      return this.userTurnSpan;
+    }
+
+    this.userTurnSpan = tracer.startSpan({
+      name: 'user_turn',
+      context: this.rootSpanContext,
+      startTime,
+    });
+
+    const participant = this.getLinkedParticipant?.();
+    if (participant) {
+      setParticipantSpanAttributes(this.userTurnSpan, participant);
+    }
+
+    if (this.sttModel) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.sttModel);
+    }
+    if (this.sttProvider) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_PROVIDER_NAME, this.sttProvider);
+    }
+
+    return this.userTurnSpan;
+  }
+
+  private userTurnContext(span: Span): Context {
+    const base = this.rootSpanContext ?? ROOT_CONTEXT;
+    return trace.setSpan(base, span);
   }
 
   private async onSTTEvent(ev: SpeechEvent) {
@@ -140,10 +506,29 @@ export class AudioRecognition {
       return;
     }
 
+    // handle interruption detection
+    // - hold the event until the ignore_user_transcript_until expires
+    // - release only relevant events
+    // - allow RECOGNITION_USAGE to pass through immediately
+
+    if (ev.type !== SpeechEventType.RECOGNITION_USAGE && this.isInterruptionEnabled) {
+      if (this.shouldHoldSttEvent(ev)) {
+        this.logger.trace(
+          { event: ev.type, ignoreUserTranscriptUntil: this.ignoreUserTranscriptUntil },
+          'holding STT event until ignore_user_transcript_until expires',
+        );
+        this.transcriptBuffer.push(ev);
+        return;
+      } else {
+        await this.flushHeldTranscripts();
+        // no return here to allow the new event to be processed normally
+      }
+    }
+
     switch (ev.type) {
       case SpeechEventType.FINAL_TRANSCRIPT:
-        this.hooks.onFinalTranscript(ev);
         const transcript = ev.alternatives?.[0]?.text;
+        const confidence = ev.alternatives?.[0]?.confidence ?? 0;
         this.lastLanguage = ev.alternatives?.[0]?.language;
 
         if (!transcript) {
@@ -151,6 +536,8 @@ export class AudioRecognition {
           return;
         }
 
+        this.hooks.onFinalTranscript(ev);
+
         this.logger.debug(
           {
             user_transcript: transcript,
@@ -162,34 +549,156 @@ export class AudioRecognition {
         this.lastFinalTranscriptTime = Date.now();
         this.audioTranscript += ` ${transcript}`;
         this.audioTranscript = this.audioTranscript.trimStart();
+        this.finalTranscriptConfidence.push(confidence);
+        const transcriptChanged = this.audioTranscript !== this.audioPreflightTranscript;
         this.audioInterimTranscript = '';
+        this.audioPreflightTranscript = '';
+
+        if (!this.vad || this.lastSpeakingTime === undefined) {
+          // vad disabled, use stt timestamp
+          // TODO: this would screw up transcription latency metrics
+          // but we'll live with it for now.
+          // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH
+          // and using that timestamp for lastSpeakingTime
+          this.lastSpeakingTime = Date.now();
+        }
 
-        if (!this.speaking) {
-          if (!this.vad) {
-            // Copied from python agents:
-            // vad disabled, use stt timestamp
-            // TODO: this would screw up transcription latency metrics
-            // but we'll live with it for now.
-            // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH
-            // and using that timestamp for _last_speaking_time
-            this.lastSpeakingTime = Date.now();
+        if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (transcriptChanged) {
+            this.logger.debug(
+              { transcript: this.audioTranscript },
+              'triggering preemptive generation (FINAL_TRANSCRIPT)',
+            );
+            this.hooks.onPreemptiveGeneration({
+              newTranscript: this.audioTranscript,
+              transcriptConfidence:
+                this.finalTranscriptConfidence.length > 0
+                  ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) /
+                    this.finalTranscriptConfidence.length
+                  : 0,
+            });
           }
 
-          if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (!this.speaking) {
             const chatCtx = this.hooks.retrieveChatCtx();
             this.logger.debug('running EOU detection on stt FINAL_TRANSCRIPT');
             this.runEOUDetection(chatCtx);
           }
         }
         break;
+      case SpeechEventType.PREFLIGHT_TRANSCRIPT:
+        this.hooks.onInterimTranscript(ev);
+        const preflightTranscript = ev.alternatives?.[0]?.text ?? '';
+        const preflightConfidence = ev.alternatives?.[0]?.confidence ?? 0;
+        const preflightLanguage = ev.alternatives?.[0]?.language;
+
+        const MIN_LANGUAGE_DETECTION_LENGTH = 5;
+        if (
+          !this.lastLanguage ||
+          (preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH)
+        ) {
+          this.lastLanguage = preflightLanguage;
+        }
+
+        if (!preflightTranscript) {
+          return;
+        }
+
+        this.logger.debug(
+          {
+            user_transcript: preflightTranscript,
+            language: this.lastLanguage,
+          },
+          'received user preflight transcript',
+        );
+
+        // still need to increment it as it's used for turn detection,
+        this.lastFinalTranscriptTime = Date.now();
+        // preflight transcript includes all pre-committed transcripts (including final transcript from the previous STT run)
+        this.audioPreflightTranscript =
+          `${this.audioTranscript} ${preflightTranscript}`.trimStart();
+        this.audioInterimTranscript = preflightTranscript;
+
+        if (!this.vad || this.lastSpeakingTime === undefined) {
+          // vad disabled, use stt timestamp
+          this.lastSpeakingTime = Date.now();
+        }
+
+        if (this.turnDetectionMode !== 'manual' || this.userTurnCommitted) {
+          const confidenceVals = [...this.finalTranscriptConfidence, preflightConfidence];
+          this.logger.debug(
+            {
+              transcript:
+                this.audioPreflightTranscript.length > 100
+                  ? this.audioPreflightTranscript.slice(0, 100) + '...'
+                  : this.audioPreflightTranscript,
+            },
+            'triggering preemptive generation (PREFLIGHT_TRANSCRIPT)',
+          );
+          this.hooks.onPreemptiveGeneration({
+            newTranscript: this.audioPreflightTranscript,
+            transcriptConfidence:
+              confidenceVals.length > 0
+                ? confidenceVals.reduce((a, b) => a + b, 0) / confidenceVals.length
+                : 0,
+          });
+        }
+        break;
       case SpeechEventType.INTERIM_TRANSCRIPT:
         this.logger.debug({ transcript: ev.alternatives?.[0]?.text }, 'interim transcript');
         this.hooks.onInterimTranscript(ev);
         this.audioInterimTranscript = ev.alternatives?.[0]?.text ?? '';
         break;
+      case SpeechEventType.START_OF_SPEECH:
+        if (this.turnDetectionMode !== 'stt') break;
+        {
+          const span = this.ensureUserTurnSpan(Date.now());
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onStartOfSpeech({
+              type: VADEventType.START_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: true,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0,
+            });
+          });
+        }
+        this.speaking = true;
+        this.lastSpeakingTime = Date.now();
+
+        this.bounceEOUTask?.cancel();
+        break;
       case SpeechEventType.END_OF_SPEECH:
         if (this.turnDetectionMode !== 'stt') break;
+        {
+          const span = this.ensureUserTurnSpan();
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onEndOfSpeech({
+              type: VADEventType.END_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: false,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0,
+            });
+          });
+        }
+        this.speaking = false;
         this.userTurnCommitted = true;
+        this.lastSpeakingTime = Date.now();
 
         if (!this.speaking) {
           const chatCtx = this.hooks.retrieveChatCtx();
@@ -199,6 +708,12 @@ export class AudioRecognition {
     }
   }
 
+  private onOverlapSpeechEvent(ev: OverlappingSpeechEvent) {
+    if (ev.isInterruption) {
+      this.hooks.onInterruption(ev);
+    }
+  }
+
   private runEOUDetection(chatCtx: ChatContext) {
     this.logger.debug(
       {
@@ -222,61 +737,132 @@ export class AudioRecognition {
       // disable EOU model if manual turn detection enabled
       this.audioTranscript && this.turnDetectionMode !== 'manual' ? this.turnDetector : undefined;
 
-    const bounceEOUTask = (lastSpeakingTime: number) => async (controller: AbortController) => {
-      let endpointingDelay = this.minEndpointingDelay;
-
-      // TODO(AJS-74): need to support actual turn detection model plugins for following code to run
-      if (turnDetector) {
-        this.logger.debug('Running turn detector model');
-        if (!turnDetector.supportsLanguage(this.lastLanguage)) {
-          this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
-        } else {
-          const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
-          this.logger.debug(
-            { endOfTurnProbability, language: this.lastLanguage },
-            'end of turn probability',
-          );
-
-          const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
-          this.logger.debug(
+    const bounceEOUTask =
+      (
+        lastSpeakingTime: number | undefined,
+        lastFinalTranscriptTime: number,
+        speechStartTime: number | undefined,
+      ) =>
+      async (controller: AbortController) => {
+        let endpointingDelay = this.minEndpointingDelay;
+
+        const userTurnSpan = this.ensureUserTurnSpan();
+        const userTurnCtx = this.userTurnContext(userTurnSpan);
+
+        if (turnDetector) {
+          await tracer.startActiveSpan(
+            async (span) => {
+              this.logger.debug('Running turn detector model');
+
+              let endOfTurnProbability = 0.0;
+              let unlikelyThreshold: number | undefined;
+
+              if (!(await turnDetector.supportsLanguage(this.lastLanguage))) {
+                this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
+              } else {
+                try {
+                  endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
+                  unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
+
+                  this.logger.debug(
+                    { endOfTurnProbability, unlikelyThreshold, language: this.lastLanguage },
+                    'end of turn probability',
+                  );
+
+                  if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
+                    endpointingDelay = this.maxEndpointingDelay;
+                  }
+                } catch (error) {
+                  this.logger.error(error, 'Error predicting end of turn');
+                }
+              }
+
+              span.setAttribute(
+                traceTypes.ATTR_CHAT_CTX,
+                JSON.stringify(chatCtx.toJSON({ excludeTimestamp: false })),
+              );
+              span.setAttribute(traceTypes.ATTR_EOU_PROBABILITY, endOfTurnProbability);
+              span.setAttribute(traceTypes.ATTR_EOU_UNLIKELY_THRESHOLD, unlikelyThreshold ?? 0);
+              span.setAttribute(traceTypes.ATTR_EOU_DELAY, endpointingDelay);
+              span.setAttribute(traceTypes.ATTR_EOU_LANGUAGE, this.lastLanguage ?? '');
+            },
             {
-              unlikelyThreshold,
-              endOfTurnProbability,
-              language: this.lastLanguage,
-              transcript: this.audioTranscript,
+              name: 'eou_detection',
+              context: userTurnCtx,
             },
-            'EOU Detection',
           );
-
-          if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
-            endpointingDelay = this.maxEndpointingDelay;
-          }
         }
-      }
 
-      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();
-      // add delay to see if there's a potential upcoming EOU task that cancels this one
-      await delay(Math.max(extraSleep, 0), { signal: controller.signal });
+        let extraSleep = endpointingDelay;
+        if (lastSpeakingTime !== undefined) {
+          extraSleep += lastSpeakingTime - Date.now();
+        }
 
-      this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');
+        if (extraSleep > 0) {
+          // add delay to see if there's a potential upcoming EOU task that cancels this one
+          await delay(Math.max(extraSleep, 0), { signal: controller.signal });
+        }
 
-      const committed = await this.hooks.onEndOfTurn({
-        newTranscript: this.audioTranscript,
-        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),
-        endOfUtteranceDelay: Date.now() - lastSpeakingTime,
-      });
+        this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');
+
+        const confidenceAvg =
+          this.finalTranscriptConfidence.length > 0
+            ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) /
+              this.finalTranscriptConfidence.length
+            : 0;
+
+        let startedSpeakingAt: number | undefined;
+        let stoppedSpeakingAt: number | undefined;
+        let transcriptionDelay: number | undefined;
+        let endOfUtteranceDelay: number | undefined;
+
+        // sometimes, we can't calculate the metrics because VAD was unreliable.
+        // in this case, we just ignore the calculation, it's better than providing likely wrong values
+        if (
+          lastFinalTranscriptTime !== 0 &&
+          lastSpeakingTime !== undefined &&
+          speechStartTime !== undefined
+        ) {
+          startedSpeakingAt = speechStartTime;
+          stoppedSpeakingAt = lastSpeakingTime;
+          transcriptionDelay = Math.max(lastFinalTranscriptTime - lastSpeakingTime, 0);
+          endOfUtteranceDelay = Date.now() - lastSpeakingTime;
+        }
 
-      if (committed) {
-        // clear the transcript if the user turn was committed
-        this.audioTranscript = '';
-      }
+        const committed = await this.hooks.onEndOfTurn({
+          newTranscript: this.audioTranscript,
+          transcriptConfidence: confidenceAvg,
+          transcriptionDelay: transcriptionDelay ?? 0,
+          endOfUtteranceDelay: endOfUtteranceDelay ?? 0,
+          startedSpeakingAt,
+          stoppedSpeakingAt,
+        });
+
+        if (committed) {
+          this._endUserTurnSpan({
+            transcript: this.audioTranscript,
+            confidence: confidenceAvg,
+            transcriptionDelay: transcriptionDelay ?? 0,
+            endOfUtteranceDelay: endOfUtteranceDelay ?? 0,
+          });
+
+          // clear the transcript if the user turn was committed
+          this.audioTranscript = '';
+          this.finalTranscriptConfidence = [];
+          this.lastSpeakingTime = undefined;
+          this.lastFinalTranscriptTime = 0;
+          this.speechStartTime = undefined;
+        }
 
-      this.userTurnCommitted = false;
-    };
+        this.userTurnCommitted = false;
+      };
 
     // cancel any existing EOU task
     this.bounceEOUTask?.cancel();
-    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));
+    // copy the values before awaiting (the values can change)
+    this.bounceEOUTask = Task.from(
+      bounceEOUTask(this.lastSpeakingTime, this.lastFinalTranscriptTime, this.speechStartTime),
+    );
 
     this.bounceEOUTask.result
       .then(() => {
@@ -364,7 +950,12 @@ export class AudioRecognition {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
             this.logger.debug('VAD task: START_OF_SPEECH');
-            this.hooks.onStartOfSpeech(ev);
+            {
+              const startTime = Date.now() - ev.speechDuration;
+              const span = this.ensureUserTurnSpan(startTime);
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onStartOfSpeech(ev));
+            }
             this.speaking = true;
 
             // Capture sample rate from the first VAD event if not already set
@@ -376,13 +967,27 @@ export class AudioRecognition {
             break;
           case VADEventType.INFERENCE_DONE:
             this.hooks.onVADInferenceDone(ev);
+            // for metrics, get the "earliest" signal of speech as possible
+            if (ev.rawAccumulatedSpeech > 0.0) {
+              this.lastSpeakingTime = Date.now();
+
+              if (this.speechStartTime === undefined) {
+                // Backdate speechStartTime to the actual start of accumulated speech.
+                // ev.rawAccumulatedSpeech is in ms (VADEvent durations are all ms in TS).
+                this.speechStartTime = Date.now() - ev.rawAccumulatedSpeech;
+              }
+            }
             break;
           case VADEventType.END_OF_SPEECH:
             this.logger.debug('VAD task: END_OF_SPEECH');
-            this.hooks.onEndOfSpeech(ev);
-            this.speaking = false;
+            {
+              const span = this.ensureUserTurnSpan();
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onEndOfSpeech(ev));
+            }
+
             // when VAD fires END_OF_SPEECH, it already waited for the silence_duration
-            this.lastSpeakingTime = Date.now() - ev.silenceDuration;
+            this.speaking = false;
 
             if (
               this.vadBaseTurnDetection ||
@@ -401,6 +1006,136 @@ export class AudioRecognition {
     }
   }
 
+  private async createInterruptionTask(
+    interruptionDetection: AdaptiveInterruptionDetector | undefined,
+    signal: AbortSignal,
+  ) {
+    if (!interruptionDetection || !this.interruptionStreamChannel) return;
+
+    let numRetries = 0;
+    const maxRetries = apiConnectDefaults.maxRetries;
+
+    while (!signal.aborted) {
+      const stream = interruptionDetection.createStream();
+      const eventReader = stream.stream().getReader();
+
+      const cleanup = async () => {
+        try {
+          signal.removeEventListener('abort', cleanup);
+          eventReader.releaseLock();
+          await stream.close();
+        } catch (e) {
+          this.logger.debug('createInterruptionTask: error during cleanup:', e);
+        }
+      };
+
+      signal.addEventListener('abort', cleanup, { once: true });
+
+      let forwardTask: Promise<void> | undefined;
+
+      try {
+        // Unlike Python where _agent_speech_started lives on `self` and survives retries,
+        // JS creates a fresh InterruptionStreamBase per retry with agentSpeechStarted = false.
+        // Re-inject the sentinel so the new stream knows the agent is mid-speech.
+        if (numRetries > 0 && this.isAgentSpeaking) {
+          await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted());
+        }
+
+        forwardTask = (async () => {
+          const inputReader = this.interruptionStreamChannel!.stream().getReader();
+          const abortPromise = waitForAbort(signal);
+
+          try {
+            while (!signal.aborted) {
+              const res = await Promise.race([inputReader.read(), abortPromise]);
+              if (!res) break;
+
+              const { value, done } = res;
+              if (done) break;
+
+              if (value instanceof AudioFrame) {
+                const frameDurationMs = (value.samplesPerChannel / value.sampleRate) * 1000;
+                this._inputStartedAt ??= Date.now() - frameDurationMs;
+              } else {
+                this._inputStartedAt ??= Date.now();
+              }
+
+              await stream.pushFrame(value);
+            }
+          } finally {
+            inputReader.releaseLock();
+          }
+        })();
+
+        const abortPromise = waitForAbort(signal);
+
+        while (!signal.aborted) {
+          const res = await Promise.race([eventReader.read(), abortPromise]);
+          if (!res) break;
+          const { done, value: ev } = res;
+          if (done) break;
+          this.onOverlapSpeechEvent(ev);
+        }
+        break;
+      } catch (e) {
+        if (signal.aborted) break;
+
+        if (isAPIError(e)) {
+          if (maxRetries === 0 || !e.retryable) {
+            interruptionDetection.emitError(
+              new InterruptionDetectionError(
+                e.message,
+                Date.now(),
+                interruptionDetection.label,
+                false,
+              ),
+            );
+            break;
+          } else if (numRetries >= maxRetries) {
+            interruptionDetection.emitError(
+              new InterruptionDetectionError(
+                `failed to detect interruption after ${numRetries} attempts`,
+                Date.now(),
+                interruptionDetection.label,
+                false,
+              ),
+            );
+            break;
+          } else {
+            const retryInterval = intervalForRetry(numRetries);
+            interruptionDetection.emitError(
+              new InterruptionDetectionError(
+                e.message,
+                Date.now(),
+                interruptionDetection.label,
+                true,
+              ),
+            );
+            this.logger.warn(
+              { model: interruptionDetection.label, attempt: numRetries },
+              `failed to detect interruption, retrying in ${retryInterval}ms`,
+            );
+            numRetries++;
+            await delay(retryInterval, { signal });
+          }
+        } else {
+          const msg = e instanceof Error ? e.message : String(e);
+          interruptionDetection.emitError(
+            new InterruptionDetectionError(msg, Date.now(), interruptionDetection.label, false),
+          );
+          this.logger.error(e, 'Error in interruption task');
+          break;
+        }
+      } finally {
+        await cleanup();
+        await forwardTask?.catch((e) => {
+          this.logger.debug({ err: e }, 'interruption task exited with error');
+        });
+      }
+    }
+    this.logger.debug('Interruption task closed');
+  }
+
   setInputAudioStream(audioStream: ReadableStream<AudioFrame>) {
     this.deferredInputStream.setSource(audioStream);
   }
@@ -412,6 +1147,8 @@ export class AudioRecognition {
   clearUserTurn() {
     this.audioTranscript = '';
     this.audioInterimTranscript = '';
+    this.audioPreflightTranscript = '';
+    this.finalTranscriptConfidence = [];
     this.userTurnCommitted = false;
 
     this.sttTask?.cancelAndWait().finally(() => {
@@ -460,19 +1197,57 @@ export class AudioRecognition {
         this.logger.debug('User turn committed');
       })
       .catch((err: unknown) => {
+        if (err instanceof Error && err.name === 'AbortError') {
+          this.logger.debug('User turn commit task cancelled');
+          return;
+        }
         this.logger.error(err, 'Error in user turn commit task:');
       });
   }
 
   async close() {
     this.detachInputAudioStream();
+    this.silenceAudioWriter.releaseLock();
     await this.commitUserTurnTask?.cancelAndWait();
     await this.sttTask?.cancelAndWait();
     await this.vadTask?.cancelAndWait();
     await this.bounceEOUTask?.cancelAndWait();
+    await this.interruptionTask?.cancelAndWait();
+    await this.interruptionStreamChannel?.close();
+  }
+
+  private _endUserTurnSpan({
+    transcript,
+    confidence,
+    transcriptionDelay,
+    endOfUtteranceDelay,
+  }: {
+    transcript: string;
+    confidence: number;
+    transcriptionDelay: number;
+    endOfUtteranceDelay: number;
+  }): void {
+    if (this.userTurnSpan) {
+      this.userTurnSpan.setAttributes({
+        [traceTypes.ATTR_USER_TRANSCRIPT]: transcript,
+        [traceTypes.ATTR_TRANSCRIPT_CONFIDENCE]: confidence,
+        [traceTypes.ATTR_TRANSCRIPTION_DELAY]: transcriptionDelay,
+        [traceTypes.ATTR_END_OF_TURN_DELAY]: endOfUtteranceDelay,
+      });
+      this.userTurnSpan.end();
+      this.userTurnSpan = undefined;
+    }
   }
 
   private get vadBaseTurnDetection() {
-    return ['vad', undefined].includes(this.turnDetectionMode);
+    if (typeof this.turnDetectionMode === 'object') {
+      return false;
+    }
+
+    if (this.turnDetectionMode === undefined || this.turnDetectionMode === 'vad') {
+      return true;
+    }
+
+    return false;
   }
 }