@livekit/agents 0.6.4 → 0.7.1

package/src/multimodal/agent_playout.ts

@@ -5,7 +5,7 @@ import type { AudioFrame } from '@livekit/rtc-node';
 import { type AudioSource } from '@livekit/rtc-node';
 import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
-import type { TranscriptionForwarder } from '../transcription.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 
 export const proto = {};
@@ -16,7 +16,7 @@ export class PlayoutHandle extends EventEmitter {
   #itemId: string;
   #contentIndex: number;
   /** @internal */
-  transcriptionFwd: TranscriptionForwarder;
+  synchronizer: TextAudioSynchronizer;
   /** @internal */
   doneFut: Future;
   /** @internal */
@@ -33,14 +33,14 @@ export class PlayoutHandle extends EventEmitter {
     sampleRate: number,
     itemId: string,
     contentIndex: number,
-    transcriptionFwd: TranscriptionForwarder,
+    synchronizer: TextAudioSynchronizer,
   ) {
     super();
     this.#audioSource = audioSource;
     this.#sampleRate = sampleRate;
     this.#itemId = itemId;
     this.#contentIndex = contentIndex;
-    this.transcriptionFwd = transcriptionFwd;
+    this.synchronizer = synchronizer;
     this.doneFut = new Future();
     this.intFut = new Future();
     this.#interrupted = false;
@@ -57,13 +57,16 @@ export class PlayoutHandle extends EventEmitter {
       return Math.floor(this.totalPlayedTime * this.#sampleRate);
     }
 
-    return Math.floor(
-      (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),
+    return Math.max(
+      0,
+      Math.floor(
+        (this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000),
+      ),
     );
   }
 
   get textChars(): number {
-    return this.transcriptionFwd.currentCharacterIndex;
+    return this.synchronizer.playedText.length;
   }
 
   get contentIndex(): number {
@@ -111,7 +114,7 @@ export class AgentPlayout extends EventEmitter {
   play(
     itemId: string,
     contentIndex: number,
-    transcriptionFwd: TranscriptionForwarder,
+    synchronizer: TextAudioSynchronizer,
     textStream: AsyncIterableQueue<string>,
     audioStream: AsyncIterableQueue<AudioFrame>,
   ): PlayoutHandle {
@@ -120,7 +123,7 @@ export class AgentPlayout extends EventEmitter {
       this.#sampleRate,
       itemId,
       contentIndex,
-      transcriptionFwd,
+      synchronizer,
     );
     this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
     return handle;
@@ -159,8 +162,9 @@ export class AgentPlayout extends EventEmitter {
                     if (cancelledText || cancelled) {
                       break;
                     }
-                    handle.transcriptionFwd.pushText(text);
+                    handle.synchronizer.pushText(text);
                   }
+                  handle.synchronizer.markTextSegmentEnd();
                   resolveText();
                 } catch (error) {
                   rejectText(error);
@@ -189,12 +193,12 @@ export class AgentPlayout extends EventEmitter {
                       break;
                     }
                     if (firstFrame) {
-                      handle.transcriptionFwd.start();
+                      handle.synchronizer.segmentPlayoutStarted();
                       this.emit('playout_started');
                       firstFrame = false;
                     }
 
-                    handle.transcriptionFwd.pushAudio(frame);
+                    handle.synchronizer.pushAudio(frame);
 
                     for (const f of bstream.write(frame.data.buffer)) {
                       handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;
@@ -208,7 +212,7 @@ export class AgentPlayout extends EventEmitter {
                       await this.#audioSource.captureFrame(f);
                     }
 
-                    handle.transcriptionFwd.markAudioComplete();
+                    handle.synchronizer.markAudioSegmentEnd();
 
                     await this.#audioSource.waitForPlayout();
                   }
@@ -233,6 +237,7 @@ export class AgentPlayout extends EventEmitter {
             handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
 
             if (handle.interrupted || captureTask.error) {
+              await handle.synchronizer.close(true);
               this.#audioSource.clearQueue(); // make sure to remove any queued frames
             }
 
@@ -241,15 +246,11 @@ export class AgentPlayout extends EventEmitter {
             }
 
             if (!firstFrame) {
-              if (!handle.interrupted) {
-                handle.transcriptionFwd.markTextComplete();
-              }
-
               this.emit('playout_stopped', handle.interrupted);
             }
 
             handle.doneFut.resolve();
-            await handle.transcriptionFwd.close(handle.interrupted);
+            await handle.synchronizer.close(false);
           }
 
           resolve();

package/src/multimodal/multimodal_agent.ts

@@ -22,7 +22,7 @@ import { AudioByteStream } from '../audio.js';
 import * as llm from '../llm/index.js';
 import { log } from '../log.js';
 import type { MultimodalLLMMetrics } from '../metrics/base.js';
-import { BasicTranscriptionForwarder } from '../transcription.js';
+import { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';
 import { findMicroTrackId } from '../utils.js';
 import { AgentPlayout, type PlayoutHandle } from './agent_playout.js';
 
@@ -190,7 +190,7 @@ export class MultimodalAgent extends EventEmitter {
         this.emit('agent_stopped_speaking');
         this.#speaking = false;
         if (this.#playingHandle) {
-          let text = this.#playingHandle.transcriptionFwd.text;
+          let text = this.#playingHandle.synchronizer.playedText;
           if (interrupted) {
             text += '…';
           }
@@ -245,17 +245,21 @@ export class MultimodalAgent extends EventEmitter {
         // openai.realtime.RealtimeContent
         if (message.contentType === 'text') return;
 
-        const trFwd = new BasicTranscriptionForwarder(
-          this.room!,
-          this.room!.localParticipant!.identity!,
-          this.#getLocalTrackSid()!,
-          message.responseId,
-        );
+        const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
+        synchronizer.on('textUpdated', (text) => {
+          this.#publishTranscription(
+            this.room!.localParticipant!.identity!,
+            this.#getLocalTrackSid()!,
+            text.text,
+            text.final,
+            text.id,
+          );
+        });
 
         const handle = this.#agentPlayout?.play(
           message.itemId,
           message.contentIndex,
-          trFwd,
+          synchronizer,
           message.textStream,
           message.audioStream,
         );

package/src/pipeline/agent_output.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import { log } from '../log.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { SynthesizeStream, type TTS } from '../tts/index.js';
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 import type { AgentPlayout, PlayoutHandle } from './agent_playout.js';
@@ -21,12 +22,20 @@ export class SynthesisHandle {
   #playHandle?: PlayoutHandle;
   intFut = new Future();
   #logger = log();
-
-  constructor(speechId: string, ttsSource: SpeechSource, agentPlayout: AgentPlayout, tts: TTS) {
+  synchronizer: TextAudioSynchronizer;
+
+  constructor(
+    speechId: string,
+    ttsSource: SpeechSource,
+    agentPlayout: AgentPlayout,
+    tts: TTS,
+    synchronizer: TextAudioSynchronizer,
+  ) {
     this.#speechId = speechId;
     this.ttsSource = ttsSource;
     this.#agentPlayout = agentPlayout;
     this.tts = tts;
+    this.synchronizer = synchronizer;
   }
 
   get speechId(): string {
@@ -51,7 +60,7 @@ export class SynthesisHandle {
       throw new Error('synthesis was interrupted');
     }
 
-    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue);
+    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue, this.synchronizer);
     return this.#playHandle;
   }
 
@@ -86,8 +95,18 @@ export class AgentOutput {
     await Promise.all(this.#tasks);
   }
 
-  synthesize(speechId: string, ttsSource: SpeechSource): SynthesisHandle {
-    const handle = new SynthesisHandle(speechId, ttsSource, this.#agentPlayout, this.#tts);
+  synthesize(
+    speechId: string,
+    ttsSource: SpeechSource,
+    synchronizer: TextAudioSynchronizer,
+  ): SynthesisHandle {
+    const handle = new SynthesisHandle(
+      speechId,
+      ttsSource,
+      this.#agentPlayout,
+      this.#tts,
+      synchronizer,
+    );
     const task = this.#synthesize(handle);
     this.#tasks.push(task);
     task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
@@ -136,6 +155,8 @@ const stringSynthesisTask = (text: string, handle: SynthesisHandle): Cancellable
 
     const ttsStream = handle.tts.stream();
     ttsStream.pushText(text);
+    handle.synchronizer.pushText(text);
+    handle.synchronizer.markTextSegmentEnd();
     ttsStream.flush();
     ttsStream.endInput();
     for await (const audio of ttsStream) {
@@ -178,8 +199,18 @@ const streamSynthesisTask = (
     for await (const text of stream) {
       fullText += text;
       if (cancelled) break;
+      handle.synchronizer.pushText(text);
       ttsStream.pushText(text);
     }
+    if (!cancelled) {
+      handle.synchronizer.markTextSegmentEnd();
+    }
+
+    // end the audio queue early if there is no actual text to turn into speech
+    if (!fullText || fullText.trim().length === 0) {
+      cancelled = true;
+      handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
+    }
     ttsStream.flush();
     ttsStream.endInput();
 

package/src/pipeline/agent_playout.ts

@@ -5,6 +5,7 @@ import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import EventEmitter from 'node:events';
 import { log } from '../log.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 import { SynthesisHandle } from './agent_output.js';
 
@@ -23,6 +24,7 @@ export class PlayoutHandle {
   #audioSource: AudioSource;
   playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
   totalPlayedTime?: number;
+  synchronizer: TextAudioSynchronizer;
   #interrupted = false;
   pushedDuration = 0;
   intFut = new Future();
@@ -32,10 +34,12 @@ export class PlayoutHandle {
     speechId: string,
     audioSource: AudioSource,
     playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+    synchronizer: TextAudioSynchronizer,
   ) {
     this.#speechId = speechId;
     this.#audioSource = audioSource;
     this.playoutSource = playoutSource;
+    this.synchronizer = synchronizer;
   }
 
   get speechId(): string {
@@ -91,12 +95,13 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
   play(
     speechId: string,
     playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+    synchronizer: TextAudioSynchronizer,
   ): PlayoutHandle {
     if (this.#closed) {
       throw new Error('source closed');
     }
 
-    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);
 
     this.#playoutTask = this.#playout(handle, this.#playoutTask);
     return handle;
@@ -109,6 +114,7 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
         handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
 
         if (handle.interrupted || captureTask.error) {
+          handle.synchronizer.close(true);
           this.#audioSource.clearQueue(); // make sure to remove any queued frames
         }
 
@@ -156,9 +162,11 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
               .child({ speechId: handle.speechId })
               .debug('started playing the first time');
             this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);
+            handle.synchronizer.segmentPlayoutStarted();
             firstFrame = false;
           }
           handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
+          handle.synchronizer.pushAudio(frame);
           await this.#audioSource.captureFrame(frame);
           await this.#audioSource.waitForPlayout();
         }
@@ -170,6 +178,7 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
         //   await this.#audioSource.waitForPlayout();
         // }
 
+        handle.synchronizer.close(false);
         resolve();
       });
 

package/src/pipeline/human_input.ts

@@ -57,6 +57,14 @@ export class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInp
     this.#subscribeToMicrophone();
   }
 
+  get participant(): RemoteParticipant {
+    return this.#participant;
+  }
+
+  get subscribedTrack(): RemoteAudioTrack | undefined {
+    return this.#subscribedTrack;
+  }
+
   #subscribeToMicrophone(): void {
     if (!this.#participant) {
       this.#logger.error('Participant is not set');

package/src/pipeline/pipeline_agent.ts

@@ -10,6 +10,7 @@ import {
   TrackSource,
 } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
 import type {
   CallableFunctionResult,
@@ -28,6 +29,7 @@ import {
   hyphenateWord,
 } from '../tokenize/basic/index.js';
 import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
+import { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';
 import type { TTS } from '../tts/index.js';
 import { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
@@ -78,6 +80,12 @@ export type VPACallbacks = {
   [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;
 };
 
+interface TurnDetector {
+  unlikelyThreshold: number;
+  supportsLanguage: (language?: string) => boolean;
+  predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;
+}
+
 export class AgentCallContext {
   #agent: VoicePipelineAgent;
   #llmStream: LLMStream;
@@ -206,6 +214,8 @@ export interface VPAOptions {
   beforeTTSCallback: BeforeTTSCallback;
   /** Options for assistant transcription. */
   transcription: AgentTranscriptionOptions;
+  /** Turn detection model to use. */
+  turnDetector?: TurnDetector;
 }
 
 const defaultVPAOptions: VPAOptions = {
@@ -238,7 +248,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
   #pendingAgentReply?: SpeechHandle;
   #agentReplyTask?: CancellablePromise<void>;
   #playingSpeech?: SpeechHandle;
-  #transcribedText = '';
+  transcribedText = '';
   #transcribedInterimText = '';
   #speechQueueOpen = new Future();
   #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();
@@ -251,6 +261,8 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
   #agentPublication?: LocalTrackPublication;
   #lastFinalTranscriptTime?: number;
   #lastSpeechTime?: number;
+  #transcriptionId?: string;
+  #agentTranscribedText = '';
 
   constructor(
     /** Voice Activity Detection instance. */
@@ -284,6 +296,8 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     this.#deferredValidation = new DeferredReplyValidation(
       this.#validateReplyIfPossible.bind(this),
       this.#opts.minEndpointingDelay,
+      this,
+      this.#opts.turnDetector,
     );
   }
 
@@ -492,14 +506,52 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
     this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
       this.#transcribedInterimText = event.alternatives![0].text;
+
+      this.#room!.localParticipant!.publishTranscription({
+        participantIdentity: this.#humanInput!.participant.identity,
+        trackSid: this.#humanInput!.subscribedTrack!.sid!,
+        segments: [
+          {
+            text: this.#transcribedInterimText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: '',
+          },
+        ],
+      });
     });
     this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
       const newTranscript = event.alternatives![0].text;
       if (!newTranscript) return;
 
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
+
       this.#lastFinalTranscriptTime = Date.now();
-      this.#transcribedText += (this.#transcribedText ? ' ' : '') + newTranscript;
+      this.transcribedText += (this.transcribedText ? ' ' : '') + newTranscript;
+
+      this.#room!.localParticipant!.publishTranscription({
+        participantIdentity: this.#humanInput!.participant.identity,
+        trackSid: this.#humanInput!.subscribedTrack!.sid!,
+        segments: [
+          {
+            text: this.transcribedText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: '',
+          },
+        ],
+      });
+      this.#transcriptionId = undefined;
 
       if (
         this.#opts.preemptiveSynthesis &&
@@ -564,7 +616,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     this.#pendingAgentReply = SpeechHandle.createAssistantReply(
       this.#opts.allowInterruptions,
       true,
-      this.#transcribedText,
+      this.transcribedText,
     );
     const newHandle = this.#pendingAgentReply;
     this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
@@ -674,7 +726,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.chatCtx.messages.push(userMsg);
       this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);
 
-      this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+      this.transcribedText = this.transcribedText.slice(userQuestion.length);
       handle.markUserCommitted();
     };
 
@@ -692,7 +744,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     }
     commitUserQuestionIfNeeded();
 
-    const collectedText = handle.synthesisHandle.text;
+    let collectedText = this.#agentTranscribedText;
     const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
     const interrupted = handle.interrupted;
 
@@ -701,7 +753,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         this.chatCtx.messages.push(...handle.extraToolsMessages);
       }
       if (interrupted) {
-        collectedText + '…';
+        collectedText += '…';
       }
 
       const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
@@ -798,6 +850,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         chatCtx,
         fncCtx: this.fncCtx,
       });
+
       const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);
       newSpeechHandle.initialize(answerLLMStream, answerSynthesis);
       handle.addNestedSpeech(newSpeechHandle);
@@ -832,6 +885,16 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     speechId: string,
     source: string | LLMStream | AsyncIterable<string>,
   ): SynthesisHandle {
+    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
+    synchronizer.on('textUpdated', (text) => {
+      this.#agentTranscribedText = text.text;
+      this.#room!.localParticipant!.publishTranscription({
+        participantIdentity: this.#room!.localParticipant!.identity,
+        trackSid: this.#agentPublication!.sid!,
+        segments: [text],
+      });
+    });
+
     if (!this.#agentOutput) {
       throw new Error('agent output should be initialized when ready');
     }
@@ -850,7 +913,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');
     }
 
-    return this.#agentOutput.synthesize(speechId, ttsSource);
+    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);
   }
 
   async #validateReplyIfPossible() {
@@ -862,7 +925,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     }
 
     if (!this.#pendingAgentReply) {
-      if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
+      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {
         return;
       }
       this.#synthesizeAgentReply();
@@ -969,6 +1032,7 @@ class DeferredReplyValidation {
   readonly PUNCTUATION = '.!?';
   readonly PUNCTUATION_REDUCE_FACTOR = 0.75;
   readonly LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech
+  readonly UNLIKELY_ENDPOINT_DELAY = 6000;
 
   #validateFunc: () => Promise<void>;
   #validatingPromise?: Promise<void>;
@@ -978,12 +1042,21 @@ class DeferredReplyValidation {
   #speaking = false;
   #endOfSpeechDelay: number;
   #finalTranscriptDelay: number;
+  #turnDetector?: TurnDetector;
+  #agent: VoicePipelineAgent;
   #abort?: AbortController;
 
-  constructor(validateFunc: () => Promise<void>, minEndpointingDelay: number) {
+  constructor(
+    validateFunc: () => Promise<void>,
+    minEndpointingDelay: number,
+    agent: VoicePipelineAgent,
+    turnDetector?: TurnDetector,
+  ) {
     this.#validateFunc = validateFunc;
     this.#endOfSpeechDelay = minEndpointingDelay;
     this.#finalTranscriptDelay = minEndpointingDelay;
+    this.#agent = agent;
+    this.#turnDetector = turnDetector;
   }
 
   get validating(): boolean {
@@ -1038,7 +1111,17 @@ class DeferredReplyValidation {
   }
 
   #run(delay: number) {
-    const runTask = async (delay: number, signal: AbortSignal) => {
+    const runTask = async (delay: number, chatCtx: ChatContext, signal: AbortSignal) => {
+      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {
+        const startTime = Date.now();
+        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);
+        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;
+        const elapsed = Date.now() - startTime;
+        if (eotProb < unlikelyThreshold) {
+          delay = this.UNLIKELY_ENDPOINT_DELAY;
+        }
+        delay = Math.max(0, delay - elapsed);
+      }
       const timeout = setTimeout(() => {
         this.#resetStates();
         this.#validateFunc();
@@ -1051,6 +1134,8 @@ class DeferredReplyValidation {
     this.#abort?.abort();
     this.#abort = new AbortController();
     this.#validatingFuture = new Future();
-    this.#validatingPromise = runTask(delay, this.#abort.signal);
+    const detectCtx = this.#agent.chatCtx.copy();
+    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });
+    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);
   }
 }

package/src/stt/stream_adapter.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
+import { log } from '../log.js';
 import type { VAD, VADStream } from '../vad.js';
 import { VADEventType } from '../vad.js';
 import type { SpeechEvent } from './stt.js';
@@ -71,13 +72,24 @@ export class StreamAdapterWrapper extends SpeechStream {
           case VADEventType.END_OF_SPEECH:
             this.output.put({ type: SpeechEventType.END_OF_SPEECH });
 
-            const event = await this.#stt.recognize(ev.frames);
-            if (!event.alternatives![0].text) {
+            try {
+              const event = await this.#stt.recognize(ev.frames);
+              if (!event.alternatives![0].text) {
+                continue;
+              }
+
+              this.output.put(event);
+              break;
+            } catch (error) {
+              let logger = log();
+              if (error instanceof Error) {
+                logger = logger.child({ error: error.message });
+              } else {
+                logger = logger.child({ error });
+              }
+              logger.error(`${this.label}: provider recognize task failed`);
               continue;
             }
-
-            this.output.put(event);
-            break;
         }
       }
     };

package/src/tokenize/basic/basic.ts

@@ -68,6 +68,8 @@ export const hyphenateWord = (word: string): string[] => {
   return hyphenator.hyphenateWord(word);
 };
 
+export { splitWords };
+
 export const tokenizeParagraphs = (text: string): string[] => {
   return splitParagraphs(text).map((tok) => tok[0]);
 };

package/src/tokenize/basic/index.ts

@@ -2,4 +2,10 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord } from './basic.js';
+export {
+  SentenceTokenizer,
+  WordTokenizer,
+  tokenizeParagraphs,
+  hyphenateWord,
+  splitWords,
+} from './basic.js';