@livekit/agents 1.1.0 → 1.2.0

package/dist/voice/audio_recognition.js

@@ -1,13 +1,27 @@
 import { AudioFrame } from "@livekit/rtc-node";
+import {
+  ROOT_CONTEXT,
+  context as otelContext,
+  trace
+} from "@opentelemetry/api";
 import { ReadableStream } from "node:stream/web";
+import { isAPIError } from "../_exceptions.js";
+import { apiConnectDefaults, intervalForRetry } from "../inference/interruption/defaults.js";
+import { InterruptionDetectionError } from "../inference/interruption/errors.js";
+import { InterruptionStreamSentinel } from "../inference/interruption/interruption_stream.js";
+import {
+} from "../inference/interruption/types.js";
 import {} from "../llm/chat_context.js";
 import { log } from "../log.js";
 import { DeferredReadableStream, isStreamReaderReleaseError } from "../stream/deferred_stream.js";
 import { IdentityTransform } from "../stream/identity_transform.js";
 import { mergeReadableStreams } from "../stream/merge_readable_streams.js";
+import { createStreamChannel } from "../stream/stream_channel.js";
 import { SpeechEventType } from "../stt/stt.js";
-import { Task, delay } from "../utils.js";
+import { traceTypes, tracer } from "../telemetry/index.js";
+import { Task, delay, waitForAbort } from "../utils.js";
 import { VADEventType } from "../vad.js";
+import { setParticipantSpanAttributes } from "./utils.js";
 class AudioRecognition {
   hooks;
   stt;
@@ -17,15 +31,23 @@ class AudioRecognition {
   minEndpointingDelay;
   maxEndpointingDelay;
   lastLanguage;
+  rootSpanContext;
+  sttModel;
+  sttProvider;
+  getLinkedParticipant;
   deferredInputStream;
   logger = log();
   lastFinalTranscriptTime = 0;
   audioTranscript = "";
   audioInterimTranscript = "";
-  lastSpeakingTime = 0;
+  audioPreflightTranscript = "";
+  finalTranscriptConfidence = [];
+  lastSpeakingTime;
+  speechStartTime;
   userTurnCommitted = false;
   speaking = false;
   sampleRate;
+  userTurnSpan;
   vadInputStream;
   sttInputStream;
   silenceAudioTransform = new IdentityTransform();
@@ -35,6 +57,15 @@ class AudioRecognition {
   commitUserTurnTask;
   vadTask;
   sttTask;
+  interruptionTask;
+  // interruption detection
+  interruptionDetection;
+  _inputStartedAt;
+  ignoreUserTranscriptUntil;
+  transcriptBuffer;
+  isInterruptionEnabled;
+  isAgentSpeaking;
+  interruptionStreamChannel;
   constructor(opts) {
     this.hooks = opts.recognitionHooks;
     this.stt = opts.stt;
@@ -44,10 +75,33 @@ class AudioRecognition {
     this.minEndpointingDelay = opts.minEndpointingDelay;
     this.maxEndpointingDelay = opts.maxEndpointingDelay;
     this.lastLanguage = void 0;
+    this.rootSpanContext = opts.rootSpanContext;
+    this.sttModel = opts.sttModel;
+    this.sttProvider = opts.sttProvider;
+    this.getLinkedParticipant = opts.getLinkedParticipant;
     this.deferredInputStream = new DeferredReadableStream();
-    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
-    this.vadInputStream = vadInputStream;
-    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);
+    this.interruptionDetection = opts.interruptionDetection;
+    this.transcriptBuffer = [];
+    this.isInterruptionEnabled = !!(opts.interruptionDetection && opts.vad);
+    this.isAgentSpeaking = false;
+    if (opts.interruptionDetection) {
+      const [vadInputStream, teedInput] = this.deferredInputStream.stream.tee();
+      const [inputStream, sttInputStream] = teedInput.tee();
+      this.vadInputStream = vadInputStream;
+      this.sttInputStream = mergeReadableStreams(
+        sttInputStream,
+        this.silenceAudioTransform.readable
+      );
+      this.interruptionStreamChannel = createStreamChannel();
+      this.interruptionStreamChannel.addStreamInput(inputStream);
+    } else {
+      const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
+      this.vadInputStream = vadInputStream;
+      this.sttInputStream = mergeReadableStreams(
+        sttInputStream,
+        this.silenceAudioTransform.readable
+      );
+    }
     this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();
   }
   /**
@@ -59,6 +113,14 @@ class AudioRecognition {
     }
     return this.audioTranscript;
   }
+  /** @internal */
+  get inputStartedAt() {
+    return this._inputStartedAt;
+  }
+  /** @internal */
+  updateOptions(options) {
+    this.turnDetectionMode = options.turnDetection;
+  }
   async start() {
     this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));
     this.vadTask.result.catch((err) => {
@@ -68,9 +130,194 @@ class AudioRecognition {
     this.sttTask.result.catch((err) => {
       this.logger.error(`Error running STT task: ${err}`);
     });
+    this.interruptionTask = Task.from(
+      ({ signal }) => this.createInterruptionTask(this.interruptionDetection, signal)
+    );
+    this.interruptionTask.result.catch((err) => {
+      this.logger.error(`Error running interruption task: ${err}`);
+    });
+  }
+  async stop() {
+    var _a, _b, _c;
+    await ((_a = this.sttTask) == null ? void 0 : _a.cancelAndWait());
+    await ((_b = this.vadTask) == null ? void 0 : _b.cancelAndWait());
+    await ((_c = this.interruptionTask) == null ? void 0 : _c.cancelAndWait());
+  }
+  async disableInterruptionDetection() {
+    var _a, _b;
+    this.isInterruptionEnabled = false;
+    this.interruptionDetection = void 0;
+    await ((_a = this.interruptionTask) == null ? void 0 : _a.cancelAndWait());
+    this.interruptionTask = void 0;
+    await ((_b = this.interruptionStreamChannel) == null ? void 0 : _b.close());
+    this.interruptionStreamChannel = void 0;
+  }
+  async onStartOfAgentSpeech() {
+    this.isAgentSpeaking = true;
+    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.agentSpeechStarted());
+  }
+  async onEndOfAgentSpeech(ignoreUserTranscriptUntil) {
+    if (!this.isInterruptionEnabled) {
+      this.isAgentSpeaking = false;
+      return;
+    }
+    const inputOpen = await this.trySendInterruptionSentinel(
+      InterruptionStreamSentinel.agentSpeechEnded()
+    );
+    if (!inputOpen) {
+      this.isAgentSpeaking = false;
+      return;
+    }
+    if (this.isAgentSpeaking) {
+      if (this.ignoreUserTranscriptUntil === void 0) {
+        this.onEndOfOverlapSpeech(Date.now());
+      }
+      this.ignoreUserTranscriptUntil = this.ignoreUserTranscriptUntil ? Math.min(ignoreUserTranscriptUntil, this.ignoreUserTranscriptUntil) : ignoreUserTranscriptUntil;
+      await this.flushHeldTranscripts();
+    }
+    this.isAgentSpeaking = false;
+  }
+  /** Start interruption inference when agent is speaking and overlap speech starts. */
+  async onStartOfOverlapSpeech(speechDuration, startedAt, userSpeakingSpan) {
+    if (this.isAgentSpeaking) {
+      this.trySendInterruptionSentinel(
+        InterruptionStreamSentinel.overlapSpeechStarted(
+          speechDuration,
+          startedAt,
+          userSpeakingSpan
+        )
+      );
+    }
+  }
+  /** End interruption inference when overlap speech ends. */
+  async onEndOfOverlapSpeech(endedAt, userSpeakingSpan) {
+    if (!this.isInterruptionEnabled) {
+      return;
+    }
+    if (userSpeakingSpan && userSpeakingSpan.isRecording()) {
+      userSpeakingSpan.setAttribute(traceTypes.ATTR_IS_INTERRUPTION, "false");
+    }
+    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.overlapSpeechEnded(endedAt));
+  }
+  /**
+   * Flush held transcripts whose *end time* is after the ignoreUserTranscriptUntil timestamp.
+   * If the event has no timestamps, we assume it is the same as the next valid event.
+   */
+  async flushHeldTranscripts() {
+    if (!this.isInterruptionEnabled || this.ignoreUserTranscriptUntil === void 0 || this.transcriptBuffer.length === 0) {
+      return;
+    }
+    if (!this._inputStartedAt) {
+      this.transcriptBuffer = [];
+      this.ignoreUserTranscriptUntil = void 0;
+      return;
+    }
+    let emitFromIndex = null;
+    let shouldFlush = false;
+    for (let i = 0; i < this.transcriptBuffer.length; i++) {
+      const ev = this.transcriptBuffer[i];
+      if (!ev || !ev.alternatives || ev.alternatives.length === 0) {
+        emitFromIndex = Math.min(emitFromIndex ?? i, i);
+        continue;
+      }
+      const firstAlternative = ev.alternatives[0];
+      if (firstAlternative.startTime === firstAlternative.endTime && firstAlternative.startTime === 0) {
+        this.transcriptBuffer = [];
+        this.ignoreUserTranscriptUntil = void 0;
+        return;
+      }
+      if (this.#alternativeEndsBeforeIgnoreWindow(firstAlternative)) {
+        emitFromIndex = null;
+      } else {
+        emitFromIndex = Math.min(emitFromIndex ?? i, i);
+        shouldFlush = true;
+        break;
+      }
+    }
+    const eventsToEmit = emitFromIndex !== null && shouldFlush ? this.transcriptBuffer.slice(emitFromIndex) : [];
+    this.transcriptBuffer = [];
+    this.ignoreUserTranscriptUntil = void 0;
+    for (const event of eventsToEmit) {
+      this.logger.trace(
+        {
+          event: event.type
+        },
+        "re-emitting held user transcript"
+      );
+      this.onSTTEvent(event);
+    }
+  }
+  #alternativeEndsBeforeIgnoreWindow(alternative) {
+    if (this.ignoreUserTranscriptUntil === void 0 || !this._inputStartedAt || alternative.startTime <= 0) {
+      return false;
+    }
+    return alternative.startTime * 1e3 + this._inputStartedAt < this.ignoreUserTranscriptUntil;
+  }
+  shouldHoldSttEvent(ev) {
+    if (!this.isInterruptionEnabled) {
+      return false;
+    }
+    if (this.isAgentSpeaking) {
+      return true;
+    }
+    if (ev.type === SpeechEventType.START_OF_SPEECH) {
+      this.ignoreUserTranscriptUntil = void 0;
+      this.transcriptBuffer = [];
+      return false;
+    }
+    if (this.ignoreUserTranscriptUntil === void 0) {
+      return false;
+    }
+    if (!ev.alternatives || ev.alternatives.length === 0) {
+      return true;
+    }
+    const alternative = ev.alternatives[0];
+    if (alternative.startTime !== alternative.endTime && this.#alternativeEndsBeforeIgnoreWindow(alternative)) {
+      return true;
+    }
+    return false;
+  }
+  async trySendInterruptionSentinel(frame) {
+    if (this.isInterruptionEnabled && this.interruptionStreamChannel && !this.interruptionStreamChannel.closed) {
+      try {
+        await this.interruptionStreamChannel.write(frame);
+        return true;
+      } catch (e) {
+        this.logger.warn(
+          `could not forward interruption sentinel: ${e instanceof Error ? e.message : String(e)}`
+        );
+      }
+    }
+    return false;
+  }
+  ensureUserTurnSpan(startTime) {
+    var _a;
+    if (this.userTurnSpan && this.userTurnSpan.isRecording()) {
+      return this.userTurnSpan;
+    }
+    this.userTurnSpan = tracer.startSpan({
+      name: "user_turn",
+      context: this.rootSpanContext,
+      startTime
+    });
+    const participant = (_a = this.getLinkedParticipant) == null ? void 0 : _a.call(this);
+    if (participant) {
+      setParticipantSpanAttributes(this.userTurnSpan, participant);
+    }
+    if (this.sttModel) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.sttModel);
+    }
+    if (this.sttProvider) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_PROVIDER_NAME, this.sttProvider);
+    }
+    return this.userTurnSpan;
+  }
+  userTurnContext(span) {
+    const base = this.rootSpanContext ?? ROOT_CONTEXT;
+    return trace.setSpan(base, span);
   }
   async onSTTEvent(ev) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r;
     if (this.turnDetectionMode === "manual" && this.userTurnCommitted && (this.bounceEOUTask === void 0 || this.bounceEOUTask.done || ev.type == SpeechEventType.INTERIM_TRANSCRIPT)) {
       this.logger.debug(
         {
@@ -83,14 +330,27 @@ class AudioRecognition {
       );
       return;
     }
+    if (ev.type !== SpeechEventType.RECOGNITION_USAGE && this.isInterruptionEnabled) {
+      if (this.shouldHoldSttEvent(ev)) {
+        this.logger.trace(
+          { event: ev.type, ignoreUserTranscriptUntil: this.ignoreUserTranscriptUntil },
+          "holding STT event until ignore_user_transcript_until expires"
+        );
+        this.transcriptBuffer.push(ev);
+        return;
+      } else {
+        await this.flushHeldTranscripts();
+      }
+    }
     switch (ev.type) {
       case SpeechEventType.FINAL_TRANSCRIPT:
-        this.hooks.onFinalTranscript(ev);
         const transcript = (_c = (_b = ev.alternatives) == null ? void 0 : _b[0]) == null ? void 0 : _c.text;
-        this.lastLanguage = (_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.language;
+        const confidence = ((_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.confidence) ?? 0;
+        this.lastLanguage = (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.language;
         if (!transcript) {
           return;
         }
+        this.hooks.onFinalTranscript(ev);
         this.logger.debug(
           {
             user_transcript: transcript,
@@ -101,26 +361,124 @@ class AudioRecognition {
         this.lastFinalTranscriptTime = Date.now();
         this.audioTranscript += ` ${transcript}`;
         this.audioTranscript = this.audioTranscript.trimStart();
+        this.finalTranscriptConfidence.push(confidence);
+        const transcriptChanged = this.audioTranscript !== this.audioPreflightTranscript;
         this.audioInterimTranscript = "";
-        if (!this.speaking) {
-          if (!this.vad) {
-            this.lastSpeakingTime = Date.now();
+        this.audioPreflightTranscript = "";
+        if (!this.vad || this.lastSpeakingTime === void 0) {
+          this.lastSpeakingTime = Date.now();
+        }
+        if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (transcriptChanged) {
+            this.logger.debug(
+              { transcript: this.audioTranscript },
+              "triggering preemptive generation (FINAL_TRANSCRIPT)"
+            );
+            this.hooks.onPreemptiveGeneration({
+              newTranscript: this.audioTranscript,
+              transcriptConfidence: this.finalTranscriptConfidence.length > 0 ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) / this.finalTranscriptConfidence.length : 0
+            });
           }
-          if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (!this.speaking) {
             const chatCtx = this.hooks.retrieveChatCtx();
             this.logger.debug("running EOU detection on stt FINAL_TRANSCRIPT");
             this.runEOUDetection(chatCtx);
           }
         }
         break;
+      case SpeechEventType.PREFLIGHT_TRANSCRIPT:
+        this.hooks.onInterimTranscript(ev);
+        const preflightTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        const preflightConfidence = ((_k = (_j = ev.alternatives) == null ? void 0 : _j[0]) == null ? void 0 : _k.confidence) ?? 0;
+        const preflightLanguage = (_m = (_l = ev.alternatives) == null ? void 0 : _l[0]) == null ? void 0 : _m.language;
+        const MIN_LANGUAGE_DETECTION_LENGTH = 5;
+        if (!this.lastLanguage || preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH) {
+          this.lastLanguage = preflightLanguage;
+        }
+        if (!preflightTranscript) {
+          return;
+        }
+        this.logger.debug(
+          {
+            user_transcript: preflightTranscript,
+            language: this.lastLanguage
+          },
+          "received user preflight transcript"
+        );
+        this.lastFinalTranscriptTime = Date.now();
+        this.audioPreflightTranscript = `${this.audioTranscript} ${preflightTranscript}`.trimStart();
+        this.audioInterimTranscript = preflightTranscript;
+        if (!this.vad || this.lastSpeakingTime === void 0) {
+          this.lastSpeakingTime = Date.now();
+        }
+        if (this.turnDetectionMode !== "manual" || this.userTurnCommitted) {
+          const confidenceVals = [...this.finalTranscriptConfidence, preflightConfidence];
+          this.logger.debug(
+            {
+              transcript: this.audioPreflightTranscript.length > 100 ? this.audioPreflightTranscript.slice(0, 100) + "..." : this.audioPreflightTranscript
+            },
+            "triggering preemptive generation (PREFLIGHT_TRANSCRIPT)"
+          );
+          this.hooks.onPreemptiveGeneration({
+            newTranscript: this.audioPreflightTranscript,
+            transcriptConfidence: confidenceVals.length > 0 ? confidenceVals.reduce((a, b) => a + b, 0) / confidenceVals.length : 0
+          });
+        }
+        break;
       case SpeechEventType.INTERIM_TRANSCRIPT:
-        this.logger.debug({ transcript: (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.text }, "interim transcript");
+        this.logger.debug({ transcript: (_o = (_n = ev.alternatives) == null ? void 0 : _n[0]) == null ? void 0 : _o.text }, "interim transcript");
         this.hooks.onInterimTranscript(ev);
-        this.audioInterimTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        this.audioInterimTranscript = ((_q = (_p = ev.alternatives) == null ? void 0 : _p[0]) == null ? void 0 : _q.text) ?? "";
+        break;
+      case SpeechEventType.START_OF_SPEECH:
+        if (this.turnDetectionMode !== "stt") break;
+        {
+          const span = this.ensureUserTurnSpan(Date.now());
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onStartOfSpeech({
+              type: VADEventType.START_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: true,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0
+            });
+          });
+        }
+        this.speaking = true;
+        this.lastSpeakingTime = Date.now();
+        (_r = this.bounceEOUTask) == null ? void 0 : _r.cancel();
         break;
       case SpeechEventType.END_OF_SPEECH:
         if (this.turnDetectionMode !== "stt") break;
+        {
+          const span = this.ensureUserTurnSpan();
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onEndOfSpeech({
+              type: VADEventType.END_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: false,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0
+            });
+          });
+        }
+        this.speaking = false;
         this.userTurnCommitted = true;
+        this.lastSpeakingTime = Date.now();
         if (!this.speaking) {
           const chatCtx = this.hooks.retrieveChatCtx();
           this.logger.debug("running EOU detection on stt END_OF_SPEECH");
@@ -128,6 +486,11 @@ class AudioRecognition {
         }
     }
   }
+  onOverlapSpeechEvent(ev) {
+    if (ev.isInterruption) {
+      this.hooks.onInterruption(ev);
+    }
+  }
   runEOUDetection(chatCtx) {
     var _a;
     this.logger.debug(
@@ -148,48 +511,94 @@ class AudioRecognition {
       // disable EOU model if manual turn detection enabled
       this.audioTranscript && this.turnDetectionMode !== "manual" ? this.turnDetector : void 0
     );
-    const bounceEOUTask = (lastSpeakingTime) => async (controller) => {
+    const bounceEOUTask = (lastSpeakingTime, lastFinalTranscriptTime, speechStartTime) => async (controller) => {
       let endpointingDelay = this.minEndpointingDelay;
+      const userTurnSpan = this.ensureUserTurnSpan();
+      const userTurnCtx = this.userTurnContext(userTurnSpan);
       if (turnDetector) {
-        this.logger.debug("Running turn detector model");
-        if (!turnDetector.supportsLanguage(this.lastLanguage)) {
-          this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
-        } else {
-          const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
-          this.logger.debug(
-            { endOfTurnProbability, language: this.lastLanguage },
-            "end of turn probability"
-          );
-          const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
-          this.logger.debug(
-            {
-              unlikelyThreshold,
-              endOfTurnProbability,
-              language: this.lastLanguage,
-              transcript: this.audioTranscript
-            },
-            "EOU Detection"
-          );
-          if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
-            endpointingDelay = this.maxEndpointingDelay;
+        await tracer.startActiveSpan(
+          async (span) => {
+            this.logger.debug("Running turn detector model");
+            let endOfTurnProbability = 0;
+            let unlikelyThreshold;
+            if (!await turnDetector.supportsLanguage(this.lastLanguage)) {
+              this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
+            } else {
+              try {
+                endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
+                unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
+                this.logger.debug(
+                  { endOfTurnProbability, unlikelyThreshold, language: this.lastLanguage },
+                  "end of turn probability"
+                );
+                if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
+                  endpointingDelay = this.maxEndpointingDelay;
+                }
+              } catch (error) {
+                this.logger.error(error, "Error predicting end of turn");
+              }
+            }
+            span.setAttribute(
+              traceTypes.ATTR_CHAT_CTX,
+              JSON.stringify(chatCtx.toJSON({ excludeTimestamp: false }))
+            );
+            span.setAttribute(traceTypes.ATTR_EOU_PROBABILITY, endOfTurnProbability);
+            span.setAttribute(traceTypes.ATTR_EOU_UNLIKELY_THRESHOLD, unlikelyThreshold ?? 0);
+            span.setAttribute(traceTypes.ATTR_EOU_DELAY, endpointingDelay);
+            span.setAttribute(traceTypes.ATTR_EOU_LANGUAGE, this.lastLanguage ?? "");
+          },
+          {
+            name: "eou_detection",
+            context: userTurnCtx
           }
-        }
+        );
+      }
+      let extraSleep = endpointingDelay;
+      if (lastSpeakingTime !== void 0) {
+        extraSleep += lastSpeakingTime - Date.now();
+      }
+      if (extraSleep > 0) {
+        await delay(Math.max(extraSleep, 0), { signal: controller.signal });
       }
-      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();
-      await delay(Math.max(extraSleep, 0), { signal: controller.signal });
       this.logger.debug({ transcript: this.audioTranscript }, "end of user turn");
+      const confidenceAvg = this.finalTranscriptConfidence.length > 0 ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) / this.finalTranscriptConfidence.length : 0;
+      let startedSpeakingAt;
+      let stoppedSpeakingAt;
+      let transcriptionDelay;
+      let endOfUtteranceDelay;
+      if (lastFinalTranscriptTime !== 0 && lastSpeakingTime !== void 0 && speechStartTime !== void 0) {
+        startedSpeakingAt = speechStartTime;
+        stoppedSpeakingAt = lastSpeakingTime;
+        transcriptionDelay = Math.max(lastFinalTranscriptTime - lastSpeakingTime, 0);
+        endOfUtteranceDelay = Date.now() - lastSpeakingTime;
+      }
       const committed = await this.hooks.onEndOfTurn({
         newTranscript: this.audioTranscript,
-        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),
-        endOfUtteranceDelay: Date.now() - lastSpeakingTime
+        transcriptConfidence: confidenceAvg,
+        transcriptionDelay: transcriptionDelay ?? 0,
+        endOfUtteranceDelay: endOfUtteranceDelay ?? 0,
+        startedSpeakingAt,
+        stoppedSpeakingAt
       });
       if (committed) {
+        this._endUserTurnSpan({
+          transcript: this.audioTranscript,
+          confidence: confidenceAvg,
+          transcriptionDelay: transcriptionDelay ?? 0,
+          endOfUtteranceDelay: endOfUtteranceDelay ?? 0
+        });
         this.audioTranscript = "";
+        this.finalTranscriptConfidence = [];
+        this.lastSpeakingTime = void 0;
+        this.lastFinalTranscriptTime = 0;
+        this.speechStartTime = void 0;
       }
       this.userTurnCommitted = false;
     };
     (_a = this.bounceEOUTask) == null ? void 0 : _a.cancel();
-    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));
+    this.bounceEOUTask = Task.from(
+      bounceEOUTask(this.lastSpeakingTime, this.lastFinalTranscriptTime, this.speechStartTime)
+    );
     this.bounceEOUTask.result.then(() => {
       this.logger.debug("EOU detection task completed");
     }).catch((err) => {
@@ -260,7 +669,12 @@ class AudioRecognition {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
             this.logger.debug("VAD task: START_OF_SPEECH");
-            this.hooks.onStartOfSpeech(ev);
+            {
+              const startTime = Date.now() - ev.speechDuration;
+              const span = this.ensureUserTurnSpan(startTime);
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onStartOfSpeech(ev));
+            }
             this.speaking = true;
             if (ev.frames.length > 0 && ev.frames[0]) {
               this.sampleRate = ev.frames[0].sampleRate;
@@ -269,12 +683,21 @@ class AudioRecognition {
             break;
           case VADEventType.INFERENCE_DONE:
             this.hooks.onVADInferenceDone(ev);
+            if (ev.rawAccumulatedSpeech > 0) {
+              this.lastSpeakingTime = Date.now();
+              if (this.speechStartTime === void 0) {
+                this.speechStartTime = Date.now() - ev.rawAccumulatedSpeech;
+              }
+            }
             break;
           case VADEventType.END_OF_SPEECH:
             this.logger.debug("VAD task: END_OF_SPEECH");
-            this.hooks.onEndOfSpeech(ev);
+            {
+              const span = this.ensureUserTurnSpan();
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onEndOfSpeech(ev));
+            }
             this.speaking = false;
-            this.lastSpeakingTime = Date.now() - ev.silenceDuration;
             if (this.vadBaseTurnDetection || this.turnDetectionMode === "stt" && this.userTurnCommitted) {
               const chatCtx = this.hooks.retrieveChatCtx();
               this.runEOUDetection(chatCtx);
@@ -288,6 +711,115 @@ class AudioRecognition {
       this.logger.debug("VAD task closed");
     }
   }
+  async createInterruptionTask(interruptionDetection, signal) {
+    if (!interruptionDetection || !this.interruptionStreamChannel) return;
+    let numRetries = 0;
+    const maxRetries = apiConnectDefaults.maxRetries;
+    while (!signal.aborted) {
+      const stream = interruptionDetection.createStream();
+      const eventReader = stream.stream().getReader();
+      const cleanup = async () => {
+        try {
+          signal.removeEventListener("abort", cleanup);
+          eventReader.releaseLock();
+          await stream.close();
+        } catch (e) {
+          this.logger.debug("createInterruptionTask: error during cleanup:", e);
+        }
+      };
+      signal.addEventListener("abort", cleanup, { once: true });
+      let forwardTask;
+      try {
+        if (numRetries > 0 && this.isAgentSpeaking) {
+          await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted());
+        }
+        forwardTask = (async () => {
+          const inputReader = this.interruptionStreamChannel.stream().getReader();
+          const abortPromise2 = waitForAbort(signal);
+          try {
+            while (!signal.aborted) {
+              const res = await Promise.race([inputReader.read(), abortPromise2]);
+              if (!res) break;
+              const { value, done } = res;
+              if (done) break;
+              if (value instanceof AudioFrame) {
+                const frameDurationMs = value.samplesPerChannel / value.sampleRate * 1e3;
+                this._inputStartedAt ??= Date.now() - frameDurationMs;
+              } else {
+                this._inputStartedAt ??= Date.now();
+              }
+              await stream.pushFrame(value);
+            }
+          } finally {
+            inputReader.releaseLock();
+          }
+        })();
+        const abortPromise = waitForAbort(signal);
+        while (!signal.aborted) {
+          const res = await Promise.race([eventReader.read(), abortPromise]);
+          if (!res) break;
+          const { done, value: ev } = res;
+          if (done) break;
+          this.onOverlapSpeechEvent(ev);
+        }
+        break;
+      } catch (e) {
+        if (signal.aborted) break;
+        if (isAPIError(e)) {
+          if (maxRetries === 0 || !e.retryable) {
+            interruptionDetection.emitError(
+              new InterruptionDetectionError(
+                e.message,
+                Date.now(),
+                interruptionDetection.label,
+                false
+              )
+            );
+            break;
+          } else if (numRetries >= maxRetries) {
+            interruptionDetection.emitError(
+              new InterruptionDetectionError(
+                `failed to detect interruption after ${numRetries} attempts`,
+                Date.now(),
+                interruptionDetection.label,
+                false
+              )
+            );
+            break;
+          } else {
+            const retryInterval = intervalForRetry(numRetries);
+            interruptionDetection.emitError(
+              new InterruptionDetectionError(
+                e.message,
+                Date.now(),
+                interruptionDetection.label,
+                true
+              )
+            );
+            this.logger.warn(
+              { model: interruptionDetection.label, attempt: numRetries },
+              `failed to detect interruption, retrying in ${retryInterval}ms`
+            );
+            numRetries++;
+            await delay(retryInterval, { signal });
+          }
+        } else {
+          const msg = e instanceof Error ? e.message : String(e);
+          interruptionDetection.emitError(
+            new InterruptionDetectionError(msg, Date.now(), interruptionDetection.label, false)
+          );
+          this.logger.error(e, "Error in interruption task");
+          break;
+        }
+      } finally {
+        await cleanup();
+        await (forwardTask == null ? void 0 : forwardTask.catch((e) => {
+          this.logger.debug({ err: e }, "interruption task exited with error");
+        }));
+      }
+    }
+    this.logger.debug("Interruption task closed");
+  }
   setInputAudioStream(audioStream) {
     this.deferredInputStream.setSource(audioStream);
   }
@@ -298,6 +830,8 @@ class AudioRecognition {
     var _a;
     this.audioTranscript = "";
     this.audioInterimTranscript = "";
+    this.audioPreflightTranscript = "";
+    this.finalTranscriptConfidence = [];
     this.userTurnCommitted = false;
     (_a = this.sttTask) == null ? void 0 : _a.cancelAndWait().finally(() => {
       this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));
@@ -332,19 +866,49 @@ class AudioRecognition {
     this.commitUserTurnTask.result.then(() => {
       this.logger.debug("User turn committed");
     }).catch((err) => {
+      if (err instanceof Error && err.name === "AbortError") {
+        this.logger.debug("User turn commit task cancelled");
+        return;
+      }
       this.logger.error(err, "Error in user turn commit task:");
     });
   }
   async close() {
-    var _a, _b, _c, _d;
+    var _a, _b, _c, _d, _e, _f;
     this.detachInputAudioStream();
+    this.silenceAudioWriter.releaseLock();
     await ((_a = this.commitUserTurnTask) == null ? void 0 : _a.cancelAndWait());
     await ((_b = this.sttTask) == null ? void 0 : _b.cancelAndWait());
     await ((_c = this.vadTask) == null ? void 0 : _c.cancelAndWait());
     await ((_d = this.bounceEOUTask) == null ? void 0 : _d.cancelAndWait());
+    await ((_e = this.interruptionTask) == null ? void 0 : _e.cancelAndWait());
+    await ((_f = this.interruptionStreamChannel) == null ? void 0 : _f.close());
+  }
+  _endUserTurnSpan({
+    transcript,
+    confidence,
+    transcriptionDelay,
+    endOfUtteranceDelay
+  }) {
+    if (this.userTurnSpan) {
+      this.userTurnSpan.setAttributes({
+        [traceTypes.ATTR_USER_TRANSCRIPT]: transcript,
+        [traceTypes.ATTR_TRANSCRIPT_CONFIDENCE]: confidence,
+        [traceTypes.ATTR_TRANSCRIPTION_DELAY]: transcriptionDelay,
+        [traceTypes.ATTR_END_OF_TURN_DELAY]: endOfUtteranceDelay
+      });
+      this.userTurnSpan.end();
+      this.userTurnSpan = void 0;
+    }
   }
   get vadBaseTurnDetection() {
-    return ["vad", void 0].includes(this.turnDetectionMode);
+    if (typeof this.turnDetectionMode === "object") {
+      return false;
+    }
+    if (this.turnDetectionMode === void 0 || this.turnDetectionMode === "vad") {
+      return true;
+    }
+    return false;
   }
 }
 export {