@livekit/agents 1.1.0-dev.0 → 1.2.0

package/src/voice/agent_activity.ts

@@ -45,7 +45,7 @@ import { STT, type STTError, type SpeechEvent } from '../stt/stt.js';
 import { recordRealtimeMetrics, traceTypes, tracer } from '../telemetry/index.js';
 import { splitWords } from '../tokenize/basic/word.js';
 import { TTS, type TTSError } from '../tts/tts.js';
-import { Future, Task, cancelAndWait, waitFor } from '../utils.js';
+import { Future, Task, cancelAndWait, isDevMode, isHosted, waitFor } from '../utils.js';
 import { VAD, type VADEvent } from '../vad.js';
 import type { Agent, ModelSettings } from './agent.js';
 import {
@@ -152,10 +152,11 @@ export class AgentActivity implements RecognitionHooks {
     this.onError(ev);
 
   private readonly onInterruptionOverlappingSpeech = (ev: OverlappingSpeechEvent): void => {
-    this.agentSession.emit(AgentSessionEventTypes.UserOverlappingSpeech, ev);
+    this.agentSession.emit(AgentSessionEventTypes.OverlappingSpeech, ev);
   };
 
   private readonly onInterruptionMetricsCollected = (ev: InterruptionMetrics): void => {
+    this.agentSession._usageCollector.collect(ev);
     this.agentSession.emit(
       AgentSessionEventTypes.MetricsCollected,
       createMetricsCollectedEvent({ metrics: ev }),
@@ -165,6 +166,13 @@ export class AgentActivity implements RecognitionHooks {
   private readonly onInterruptionError = (ev: InterruptionDetectionError): void => {
     const errorEvent = createErrorEvent(ev, this.interruptionDetector);
     this.agentSession.emit(AgentSessionEventTypes.Error, errorEvent);
+
+    if (!ev.recoverable) {
+      this.agentSession._onError(ev);
+      this.fallbackToVadInterruption();
+      return;
+    }
+
     this.agentSession._onError(ev);
   };
 
@@ -390,8 +398,12 @@ export class AgentActivity implements RecognitionHooks {
       turnDetector: typeof this.turnDetection === 'string' ? undefined : this.turnDetection,
       turnDetectionMode: this.turnDetectionMode,
       interruptionDetection: this.interruptionDetector,
-      minEndpointingDelay: this.agentSession.options.turnHandling.endpointing.minDelay,
-      maxEndpointingDelay: this.agentSession.options.turnHandling.endpointing.maxDelay,
+      minEndpointingDelay:
+        this.agent.turnHandling?.endpointing?.minDelay ??
+        this.agentSession.sessionOptions.turnHandling.endpointing.minDelay,
+      maxEndpointingDelay:
+        this.agent.turnHandling?.endpointing?.maxDelay ??
+        this.agentSession.sessionOptions.turnHandling.endpointing.maxDelay,
       rootSpanContext: this.agentSession.rootSpanContext,
       sttModel: this.stt?.label,
       sttProvider: this.getSttProvider(),
@@ -464,8 +476,10 @@ export class AgentActivity implements RecognitionHooks {
   }
 
   get allowInterruptions(): boolean {
-    // TODO(AJS-51): Allow options to be defined in Agent class
-    return this.agentSession.options.turnHandling.interruption?.mode !== false;
+    return (
+      this.agent.turnHandling?.interruption?.enabled ??
+      this.agentSession.sessionOptions.turnHandling.interruption.enabled
+    );
   }
 
   get useTtsAlignedTranscript(): boolean {
@@ -474,10 +488,27 @@ export class AgentActivity implements RecognitionHooks {
   }
 
   get turnDetection(): TurnDetectionMode | undefined {
-    // TODO(brian): prioritize using agent.turn_detection
-    return this.agentSession.turnDetection;
+    return this.agent.turnHandling?.turnDetection ?? this.agentSession.turnDetection;
+  }
+
+  get turnHandling() {
+    return this.agent.turnHandling ?? this.agentSession.sessionOptions.turnHandling;
   }
 
+  // get minEndpointingDelay(): number {
+  //   return (
+  //     this.agent.turnHandling?.endpointing?.minDelay ??
+  //     this.agentSession.sessionOptions.turnHandling.endpointing.minDelay
+  //   );
+  // }
+
+  // get maxEndpointingDelay(): number {
+  //   return (
+  //     this.agent.turnHandling?.endpointing?.maxDelay ??
+  //     this.agentSession.sessionOptions.turnHandling.endpointing.maxDelay
+  //   );
+  // }
+
   get toolCtx(): ToolContext {
     return this.agent.toolCtx;
   }
@@ -569,16 +600,21 @@ export class AgentActivity implements RecognitionHooks {
     });
 
     this.audioStreamId = this.audioStream.addInputStream(audioStream);
-    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream
-      .pipeThrough(aecWarmupAudioFilter)
-      .tee();
 
-    if (this.realtimeSession) {
+    if (this.realtimeSession && this.audioRecognition) {
+      const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream
+        .pipeThrough(aecWarmupAudioFilter)
+        .tee();
       this.realtimeSession.setInputAudioStream(realtimeAudioStream);
-    }
-
-    if (this.audioRecognition) {
       this.audioRecognition.setInputAudioStream(recognitionAudioStream);
+    } else if (this.realtimeSession) {
+      this.realtimeSession.setInputAudioStream(
+        this.audioStream.stream.pipeThrough(aecWarmupAudioFilter),
+      );
+    } else if (this.audioRecognition) {
+      this.audioRecognition.setInputAudioStream(
+        this.audioStream.stream.pipeThrough(aecWarmupAudioFilter),
+      );
     }
   }
 
@@ -693,6 +729,8 @@ export class AgentActivity implements RecognitionHooks {
       }
     }
 
+    this.agentSession._usageCollector.collect(ev);
+
     this.agentSession.emit(
       AgentSessionEventTypes.MetricsCollected,
       createMetricsCollectedEvent({ metrics: ev }),
@@ -828,7 +866,10 @@ export class AgentActivity implements RecognitionHooks {
       // Subtract both speechDuration and inferenceDuration to correct for VAD model latency.
       speechStartTime = speechStartTime - ev.speechDuration - ev.inferenceDuration;
     }
-    this.agentSession._updateUserState('speaking', speechStartTime);
+    this.agentSession._updateUserState('speaking', {
+      lastSpeakingTime: speechStartTime,
+      otelContext: otelContext.active(),
+    });
     if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
       // Pass speechStartTime as the absolute startedAt timestamp.
       this.audioRecognition.onStartOfOverlapSpeech(
@@ -852,7 +893,10 @@ export class AgentActivity implements RecognitionHooks {
         this.agentSession._userSpeakingSpan,
       );
     }
-    this.agentSession._updateUserState('listening', speechEndTime);
+    this.agentSession._updateUserState('listening', {
+      lastSpeakingTime: speechEndTime,
+      otelContext: otelContext.active(),
+    });
   }
 
   onVADInferenceDone(ev: VADEvent): void {
@@ -861,7 +905,9 @@ export class AgentActivity implements RecognitionHooks {
       return;
     }
 
-    if (ev.speechDuration >= this.agentSession.options.turnHandling.interruption?.minDuration) {
+    if (
+      ev.speechDuration >= this.agentSession.sessionOptions.turnHandling.interruption?.minDuration
+    ) {
       this.interruptByAudioActivity();
     }
   }
@@ -887,7 +933,7 @@ export class AgentActivity implements RecognitionHooks {
     // - This ensures consistent behavior across all interruption scenarios
     if (
       this.stt &&
-      this.agentSession.options.turnHandling.interruption?.minWords > 0 &&
+      this.agentSession.sessionOptions.turnHandling.interruption?.minWords > 0 &&
       this.audioRecognition
     ) {
       const text = this.audioRecognition.currentTranscript;
@@ -899,7 +945,7 @@ export class AgentActivity implements RecognitionHooks {
 
       // Only allow interruption if word count meets or exceeds minInterruptionWords
       // This applies to all cases: empty strings, partial speech, and full speech
-      if (wordCount < this.agentSession.options.turnHandling.interruption?.minWords) {
+      if (wordCount < this.agentSession.sessionOptions.turnHandling.interruption?.minWords) {
         return;
       }
     }
@@ -924,7 +970,7 @@ export class AgentActivity implements RecognitionHooks {
     this.restoreInterruptionByAudioActivity();
     this.interruptByAudioActivity();
     if (this.audioRecognition) {
-      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.timestamp);
+      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt);
     }
   }
 
@@ -982,7 +1028,7 @@ export class AgentActivity implements RecognitionHooks {
 
   onPreemptiveGeneration(info: PreemptiveGenerationInfo): void {
     if (
-      !this.agentSession.options.preemptiveGeneration ||
+      !this.agentSession.sessionOptions.preemptiveGeneration ||
       this.schedulingPaused ||
       (this._currentSpeech !== undefined && !this._currentSpeech.interrupted) ||
       !(this.llm instanceof LLM)
@@ -1099,16 +1145,17 @@ export class AgentActivity implements RecognitionHooks {
       this._currentSpeech &&
       this._currentSpeech.allowInterruptions &&
       !this._currentSpeech.interrupted &&
-      this.agentSession.options.turnHandling.interruption?.minWords > 0
+      this.agentSession.sessionOptions.turnHandling.interruption?.minWords > 0
     ) {
       const wordCount = splitWords(info.newTranscript, true).length;
-      if (wordCount < this.agentSession.options.turnHandling.interruption?.minWords) {
+      if (wordCount < this.agentSession.sessionOptions.turnHandling.interruption?.minWords) {
         // avoid interruption if the new_transcript contains fewer words than minInterruptionWords
         this.cancelPreemptiveGeneration();
         this.logger.info(
           {
             wordCount,
-            minInterruptionWords: this.agentSession.options.turnHandling.interruption.minWords,
+            minInterruptionWords:
+              this.agentSession.sessionOptions.turnHandling.interruption.minWords,
           },
           'skipping user input, word count below minimum interruption threshold',
         );
@@ -1148,9 +1195,19 @@ export class AgentActivity implements RecognitionHooks {
           throw new Error('Speech queue is empty');
         }
         const speechHandle = heapItem[2];
+
+        // Skip speech handles that were already interrupted/done before being
+        // picked up from the queue (e.g. interrupted during shutdown before the
+        // main loop had a chance to process them). Calling _authorizeGeneration
+        // on a done handle would create a generation Future that nobody resolves,
+        // causing the main loop to hang forever.
+        if (speechHandle.interrupted || speechHandle.done()) {
+          continue;
+        }
+
         this._currentSpeech = speechHandle;
         speechHandle._authorizeGeneration();
-        await speechHandle._waitForGeneration();
+        await speechHandle.waitIfNotInterrupted([speechHandle._waitForGeneration()]);
         this._currentSpeech = undefined;
       }
 
@@ -1344,7 +1401,24 @@ export class AgentActivity implements RecognitionHooks {
 
     this.realtimeSession?.interrupt();
 
-    if (currentSpeech === undefined) {
+    if (force) {
+      // Force-interrupt (used during shutdown): cancel all speech tasks so they
+      // don't block on I/O that will never complete (e.g. audioOutput.waitForPlayout()
+      // when the room is disconnected). Mark the current speech as done immediately
+      // so the interrupt future resolves without waiting for tasks to finish.
+      // Clear the queue so mainTask doesn't dequeue already-interrupted handles
+      // and hang on _waitForGeneration() (the generation future created by
+      // _authorizeGeneration would never resolve since _markDone is a no-op
+      // once doneFut is already settled).
+      for (const task of this.speechTasks) {
+        task.cancel();
+      }
+      if (currentSpeech && !currentSpeech.done()) {
+        currentSpeech._markDone();
+      }
+      this.speechQueue.clear();
+      future.resolve();
+    } else if (currentSpeech === undefined) {
       future.resolve();
     } else {
       currentSpeech.addDoneCallback(() => {
@@ -1942,9 +2016,7 @@ export class AgentActivity implements RecognitionHooks {
       }
 
       replyAbortController.abort();
-      await Promise.allSettled(
-        tasks.map((task) => task.cancelAndWait(AgentActivity.REPLY_TASK_CANCEL_TIMEOUT)),
-      );
+      await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
 
       let forwardedText = textOut?.text || '';
 
@@ -2038,7 +2110,7 @@ export class AgentActivity implements RecognitionHooks {
     if (toolOutput.output.length === 0) return;
 
     // important: no agent output should be used after this point
-    const { maxToolSteps } = this.agentSession.options;
+    const { maxToolSteps } = this.agentSession.sessionOptions;
     if (speechHandle.numSteps >= maxToolSteps) {
       this.logger.warn(
         { speech_id: speechHandle.id, max_tool_steps: maxToolSteps },
@@ -2505,7 +2577,7 @@ export class AgentActivity implements RecognitionHooks {
     }
 
     // important: no agent ouput should be used after this point
-    const { maxToolSteps } = this.agentSession.options;
+    const { maxToolSteps } = this.agentSession.sessionOptions;
     if (speechHandle.numSteps >= maxToolSteps) {
       this.logger.warn(
         { speech_id: speechHandle.id, max_tool_steps: maxToolSteps },
@@ -2793,16 +2865,20 @@ export class AgentActivity implements RecognitionHooks {
     const unlock = await this.lock.lock();
     try {
       this.cancelPreemptiveGeneration();
+
+      await cancelAndWait(Array.from(this.speechTasks), AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
+
+      if (this._currentSpeech && !this._currentSpeech.done()) {
+        this._currentSpeech._markDone();
+      }
+
       await this._closeSessionResources();
 
       if (this._mainTask) {
         await this._mainTask.cancelAndWait();
       }
       if (this.interruptionDetector) {
-        this.interruptionDetector.off(
-          'user_overlapping_speech',
-          this.onInterruptionOverlappingSpeech,
-        );
+        this.interruptionDetector.off('overlapping_speech', this.onInterruptionOverlappingSpeech);
         this.interruptionDetector.off('metrics_collected', this.onInterruptionMetricsCollected);
         this.interruptionDetector.off('error', this.onInterruptionError);
       }
@@ -2814,8 +2890,8 @@ export class AgentActivity implements RecognitionHooks {
   }
 
   private resolveInterruptionDetector(): AdaptiveInterruptionDetector | undefined {
-    const interruptionDetection =
-      this.agent.interruptionDetection ?? this.agentSession.interruptionDetection;
+    const agentInterruptionDetection = this.agent.turnHandling?.interruption?.mode;
+    const sessionInterruptionDetection = this.agentSession.interruptionDetection;
     if (
       !(
         this.stt &&
@@ -2827,25 +2903,43 @@ export class AgentActivity implements RecognitionHooks {
         !(this.llm instanceof RealtimeModel)
       )
     ) {
-      if (interruptionDetection === 'adaptive') {
+      if (
+        agentInterruptionDetection === 'adaptive' ||
+        sessionInterruptionDetection === 'adaptive'
+      ) {
         this.logger.warn(
           "interruptionDetection is provided, but it's not compatible with the current configuration and will be disabled",
         );
-        return undefined;
       }
+      return undefined;
+    }
+
+    if (!this.allowInterruptions) {
+      return undefined;
+    }
+
+    if (agentInterruptionDetection === 'vad') {
+      return undefined;
+    }
+
+    if (sessionInterruptionDetection === 'vad') {
+      return undefined;
     }
 
     if (
-      (interruptionDetection !== undefined && interruptionDetection === false) ||
-      interruptionDetection === 'vad'
+      agentInterruptionDetection === undefined &&
+      sessionInterruptionDetection === undefined &&
+      !isHosted() &&
+      !isDevMode()
     ) {
+      this.logger.info('adaptive interruption is disabled by default in production mode');
       return undefined;
     }
 
     try {
       const detector = new AdaptiveInterruptionDetector();
 
-      detector.on('user_overlapping_speech', this.onInterruptionOverlappingSpeech);
+      detector.on('overlapping_speech', this.onInterruptionOverlappingSpeech);
       detector.on('metrics_collected', this.onInterruptionMetricsCollected);
       detector.on('error', this.onInterruptionError);
 
@@ -2860,6 +2954,30 @@ export class AgentActivity implements RecognitionHooks {
     this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
   }
 
+  private fallbackToVadInterruption(): void {
+    if (!this.isInterruptionDetectionEnabled) return;
+
+    this.isInterruptionDetectionEnabled = false;
+    this.restoreInterruptionByAudioActivity();
+
+    if (this.interruptionDetector) {
+      this.interruptionDetector.off('overlapping_speech', this.onInterruptionOverlappingSpeech);
+      this.interruptionDetector.off('metrics_collected', this.onInterruptionMetricsCollected);
+      this.interruptionDetector.off('error', this.onInterruptionError);
+      this.interruptionDetector = undefined;
+    }
+
+    if (this.audioRecognition) {
+      this.audioRecognition.disableInterruptionDetection().catch((err) => {
+        this.logger.warn({ err }, 'error while disabling interruption detection');
+      });
+    }
+
+    this.logger.warn(
+      'adaptive interruption disabled due to unrecoverable error, falling back to VAD-based interruption',
+    );
+  }
+
   private async _closeSessionResources(): Promise<void> {
     // Unregister event handlers to prevent duplicate metrics
     if (this.llm instanceof LLM) {