@livekit/agents 0.6.4 → 0.7.0

package/src/pipeline/pipeline_agent.ts

@@ -10,6 +10,7 @@ import {
   TrackSource,
 } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
 import type {
   CallableFunctionResult,
@@ -28,6 +29,7 @@ import {
   hyphenateWord,
 } from '../tokenize/basic/index.js';
 import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
+import { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';
 import type { TTS } from '../tts/index.js';
 import { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
@@ -78,6 +80,12 @@ export type VPACallbacks = {
   [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;
 };
 
+interface TurnDetector {
+  unlikelyThreshold: number;
+  supportsLanguage: (language?: string) => boolean;
+  predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;
+}
+
 export class AgentCallContext {
   #agent: VoicePipelineAgent;
   #llmStream: LLMStream;
@@ -206,6 +214,8 @@ export interface VPAOptions {
   beforeTTSCallback: BeforeTTSCallback;
   /** Options for assistant transcription. */
   transcription: AgentTranscriptionOptions;
+  /** Turn detection model to use. */
+  turnDetector?: TurnDetector;
 }
 
 const defaultVPAOptions: VPAOptions = {
@@ -238,7 +248,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
   #pendingAgentReply?: SpeechHandle;
   #agentReplyTask?: CancellablePromise<void>;
   #playingSpeech?: SpeechHandle;
-  #transcribedText = '';
+  transcribedText = '';
   #transcribedInterimText = '';
   #speechQueueOpen = new Future();
   #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();
@@ -251,6 +261,8 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
   #agentPublication?: LocalTrackPublication;
   #lastFinalTranscriptTime?: number;
   #lastSpeechTime?: number;
+  #transcriptionId?: string;
+  #agentTranscribedText = '';
 
   constructor(
     /** Voice Activity Detection instance. */
@@ -284,6 +296,8 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     this.#deferredValidation = new DeferredReplyValidation(
       this.#validateReplyIfPossible.bind(this),
       this.#opts.minEndpointingDelay,
+      this,
+      this.#opts.turnDetector,
     );
   }
 
@@ -492,14 +506,52 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
     this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
       this.#transcribedInterimText = event.alternatives![0].text;
+
+      this.#room!.localParticipant!.publishTranscription({
+        participantIdentity: this.#humanInput!.participant.identity,
+        trackSid: this.#humanInput!.subscribedTrack!.sid!,
+        segments: [
+          {
+            text: this.#transcribedInterimText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: '',
+          },
+        ],
+      });
     });
     this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
       const newTranscript = event.alternatives![0].text;
       if (!newTranscript) return;
 
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
+
       this.#lastFinalTranscriptTime = Date.now();
-      this.#transcribedText += (this.#transcribedText ? ' ' : '') + newTranscript;
+      this.transcribedText += (this.transcribedText ? ' ' : '') + newTranscript;
+
+      this.#room!.localParticipant!.publishTranscription({
+        participantIdentity: this.#humanInput!.participant.identity,
+        trackSid: this.#humanInput!.subscribedTrack!.sid!,
+        segments: [
+          {
+            text: this.transcribedText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: '',
+          },
+        ],
+      });
+      this.#transcriptionId = undefined;
 
       if (
         this.#opts.preemptiveSynthesis &&
@@ -564,7 +616,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     this.#pendingAgentReply = SpeechHandle.createAssistantReply(
       this.#opts.allowInterruptions,
       true,
-      this.#transcribedText,
+      this.transcribedText,
     );
     const newHandle = this.#pendingAgentReply;
     this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
@@ -674,7 +726,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.chatCtx.messages.push(userMsg);
       this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);
 
-      this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+      this.transcribedText = this.transcribedText.slice(userQuestion.length);
       handle.markUserCommitted();
     };
 
@@ -692,7 +744,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     }
     commitUserQuestionIfNeeded();
 
-    const collectedText = handle.synthesisHandle.text;
+    let collectedText = this.#agentTranscribedText;
     const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
     const interrupted = handle.interrupted;
 
@@ -701,7 +753,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         this.chatCtx.messages.push(...handle.extraToolsMessages);
       }
       if (interrupted) {
-        collectedText + '…';
+        collectedText += '…';
       }
 
       const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
@@ -798,6 +850,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         chatCtx,
         fncCtx: this.fncCtx,
       });
+
       const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);
       newSpeechHandle.initialize(answerLLMStream, answerSynthesis);
       handle.addNestedSpeech(newSpeechHandle);
@@ -832,6 +885,16 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     speechId: string,
     source: string | LLMStream | AsyncIterable<string>,
   ): SynthesisHandle {
+    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
+    synchronizer.on('textUpdated', (text) => {
+      this.#agentTranscribedText = text.text;
+      this.#room!.localParticipant!.publishTranscription({
+        participantIdentity: this.#room!.localParticipant!.identity,
+        trackSid: this.#agentPublication!.sid!,
+        segments: [text],
+      });
+    });
+
     if (!this.#agentOutput) {
       throw new Error('agent output should be initialized when ready');
     }
@@ -850,7 +913,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');
     }
 
-    return this.#agentOutput.synthesize(speechId, ttsSource);
+    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);
   }
 
   async #validateReplyIfPossible() {
@@ -862,7 +925,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     }
 
     if (!this.#pendingAgentReply) {
-      if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
+      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {
         return;
       }
       this.#synthesizeAgentReply();
@@ -969,6 +1032,7 @@ class DeferredReplyValidation {
   readonly PUNCTUATION = '.!?';
   readonly PUNCTUATION_REDUCE_FACTOR = 0.75;
   readonly LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech
+  readonly UNLIKELY_ENDPOINT_DELAY = 6000;
 
   #validateFunc: () => Promise<void>;
   #validatingPromise?: Promise<void>;
@@ -978,12 +1042,21 @@ class DeferredReplyValidation {
   #speaking = false;
   #endOfSpeechDelay: number;
   #finalTranscriptDelay: number;
+  #turnDetector?: TurnDetector;
+  #agent: VoicePipelineAgent;
   #abort?: AbortController;
 
-  constructor(validateFunc: () => Promise<void>, minEndpointingDelay: number) {
+  constructor(
+    validateFunc: () => Promise<void>,
+    minEndpointingDelay: number,
+    agent: VoicePipelineAgent,
+    turnDetector?: TurnDetector,
+  ) {
     this.#validateFunc = validateFunc;
     this.#endOfSpeechDelay = minEndpointingDelay;
     this.#finalTranscriptDelay = minEndpointingDelay;
+    this.#agent = agent;
+    this.#turnDetector = turnDetector;
   }
 
   get validating(): boolean {
@@ -1038,7 +1111,17 @@ class DeferredReplyValidation {
   }
 
   #run(delay: number) {
-    const runTask = async (delay: number, signal: AbortSignal) => {
+    const runTask = async (delay: number, chatCtx: ChatContext, signal: AbortSignal) => {
+      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {
+        const startTime = Date.now();
+        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);
+        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;
+        const elapsed = Date.now() - startTime;
+        if (eotProb < unlikelyThreshold) {
+          delay = this.UNLIKELY_ENDPOINT_DELAY;
+        }
+        delay = Math.max(0, delay - elapsed);
+      }
       const timeout = setTimeout(() => {
         this.#resetStates();
         this.#validateFunc();
@@ -1051,6 +1134,8 @@ class DeferredReplyValidation {
     this.#abort?.abort();
     this.#abort = new AbortController();
     this.#validatingFuture = new Future();
-    this.#validatingPromise = runTask(delay, this.#abort.signal);
+    const detectCtx = this.#agent.chatCtx.copy();
+    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });
+    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);
   }
 }

package/src/stt/stream_adapter.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
+import { log } from '../log.js';
 import type { VAD, VADStream } from '../vad.js';
 import { VADEventType } from '../vad.js';
 import type { SpeechEvent } from './stt.js';
@@ -71,13 +72,24 @@ export class StreamAdapterWrapper extends SpeechStream {
           case VADEventType.END_OF_SPEECH:
             this.output.put({ type: SpeechEventType.END_OF_SPEECH });
 
-            const event = await this.#stt.recognize(ev.frames);
-            if (!event.alternatives![0].text) {
+            try {
+              const event = await this.#stt.recognize(ev.frames);
+              if (!event.alternatives![0].text) {
+                continue;
+              }
+
+              this.output.put(event);
+              break;
+            } catch (error) {
+              let logger = log();
+              if (error instanceof Error) {
+                logger = logger.child({ error: error.message });
+              } else {
+                logger = logger.child({ error });
+              }
+              logger.error(`${this.label}: provider recognize task failed`);
               continue;
             }
-
-            this.output.put(event);
-            break;
         }
       }
     };

package/src/tokenize/basic/basic.ts

@@ -68,6 +68,8 @@ export const hyphenateWord = (word: string): string[] => {
   return hyphenator.hyphenateWord(word);
 };
 
+export { splitWords };
+
 export const tokenizeParagraphs = (text: string): string[] => {
   return splitParagraphs(text).map((tok) => tok[0]);
 };

package/src/tokenize/basic/index.ts

@@ -2,4 +2,10 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord } from './basic.js';
+export {
+  SentenceTokenizer,
+  WordTokenizer,
+  tokenizeParagraphs,
+  hyphenateWord,
+  splitWords,
+} from './basic.js';

package/src/tokenize/token_stream.ts

@@ -44,12 +44,15 @@ export class BufferedTokenStream implements AsyncIterableIterator<TokenData> {
       if (this.#outBuf) this.#outBuf += ' ';
 
       const tok = tokens.shift()!;
-      let tokText = tok as string;
-      if (tok.length > 1 && typeof tok[1] === 'number') {
+      let tokText: string;
+      if (Array.isArray(tok)) {
         tokText = tok[0];
+      } else {
+        tokText = tok;
       }
 
       this.#outBuf += tokText;
+
       if (this.#outBuf.length >= this.#minTokenLength) {
         this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
         this.#outBuf = '';
@@ -76,7 +79,7 @@ export class BufferedTokenStream implements AsyncIterableIterator<TokenData> {
       if (tokens) {
         if (this.#outBuf) this.#outBuf += ' ';
 
-        if (typeof tokens[0] !== 'string') {
+        if (Array.isArray(tokens[0])) {
           this.#outBuf += tokens.map((tok) => tok[0]).join(' ');
         } else {
           this.#outBuf += tokens.join(' ');