@livekit/agents 0.4.6 → 0.5.1

package/dist/pipeline/pipeline_agent.js

@@ -1,671 +1,615 @@
-var _a;
-import { AudioSource, LocalAudioTrack, RoomEvent, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
-import EventEmitter from 'node:events';
-import { LLMStream } from '../llm/index.js';
-import { ChatContext, ChatMessage, ChatRole } from '../llm/index.js';
-import { log } from '../log.js';
-import { StreamAdapter as STTStreamAdapter } from '../stt/index.js';
-import { SentenceTokenizer as BasicSentenceTokenizer, WordTokenizer as BasicWordTokenizer, hyphenateWord, } from '../tokenize/basic/index.js';
-import { StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
-import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
-import { AgentOutput } from './agent_output.js';
-import { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';
-import { HumanInput, HumanInputEvent } from './human_input.js';
-import { SpeechHandle } from './speech_handle.js';
-export const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';
-export var VPAEvent;
-(function (VPAEvent) {
-    VPAEvent[VPAEvent["USER_STARTED_SPEAKING"] = 0] = "USER_STARTED_SPEAKING";
-    VPAEvent[VPAEvent["USER_STOPPED_SPEAKING"] = 1] = "USER_STOPPED_SPEAKING";
-    VPAEvent[VPAEvent["AGENT_STARTED_SPEAKING"] = 2] = "AGENT_STARTED_SPEAKING";
-    VPAEvent[VPAEvent["AGENT_STOPPED_SPEAKING"] = 3] = "AGENT_STOPPED_SPEAKING";
-    VPAEvent[VPAEvent["USER_SPEECH_COMMITTED"] = 4] = "USER_SPEECH_COMMITTED";
-    VPAEvent[VPAEvent["AGENT_SPEECH_COMMITTED"] = 5] = "AGENT_SPEECH_COMMITTED";
-    VPAEvent[VPAEvent["AGENT_SPEECH_INTERRUPTED"] = 6] = "AGENT_SPEECH_INTERRUPTED";
-    VPAEvent[VPAEvent["FUNCTION_CALLS_COLLECTED"] = 7] = "FUNCTION_CALLS_COLLECTED";
-    VPAEvent[VPAEvent["FUNCTION_CALLS_FINISHED"] = 8] = "FUNCTION_CALLS_FINISHED";
-})(VPAEvent || (VPAEvent = {}));
-export class AgentCallContext {
-    #agent;
-    #llmStream;
-    #metadata = new Map();
-    static #current;
-    constructor(agent, llmStream) {
-        this.#agent = agent;
-        this.#llmStream = llmStream;
-        AgentCallContext.#current = this;
-    }
-    static getCurrent() {
-        return AgentCallContext.#current;
-    }
-    get agent() {
-        return this.#agent;
-    }
-    storeMetadata(key, value) {
-        this.#metadata.set(key, value);
-    }
-    getMetadata(key, orDefault = undefined) {
-        return this.#metadata.get(key) || orDefault;
-    }
-    get llmStream() {
-        return this.#llmStream;
-    }
+import {
+  AudioSource,
+  LocalAudioTrack,
+  RoomEvent,
+  TrackPublishOptions,
+  TrackSource
+} from "@livekit/rtc-node";
+import EventEmitter from "node:events";
+import { LLMStream } from "../llm/index.js";
+import { ChatContext, ChatMessage, ChatRole } from "../llm/index.js";
+import { log } from "../log.js";
+import { StreamAdapter as STTStreamAdapter } from "../stt/index.js";
+import {
+  SentenceTokenizer as BasicSentenceTokenizer,
+  WordTokenizer as BasicWordTokenizer,
+  hyphenateWord
+} from "../tokenize/basic/index.js";
+import { StreamAdapter as TTSStreamAdapter } from "../tts/index.js";
+import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from "../utils.js";
+import { AgentOutput } from "./agent_output.js";
+import { AgentPlayout, AgentPlayoutEvent } from "./agent_playout.js";
+import { HumanInput, HumanInputEvent } from "./human_input.js";
+import { SpeechHandle } from "./speech_handle.js";
+const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
+var VPAEvent = /* @__PURE__ */ ((VPAEvent2) => {
+  VPAEvent2[VPAEvent2["USER_STARTED_SPEAKING"] = 0] = "USER_STARTED_SPEAKING";
+  VPAEvent2[VPAEvent2["USER_STOPPED_SPEAKING"] = 1] = "USER_STOPPED_SPEAKING";
+  VPAEvent2[VPAEvent2["AGENT_STARTED_SPEAKING"] = 2] = "AGENT_STARTED_SPEAKING";
+  VPAEvent2[VPAEvent2["AGENT_STOPPED_SPEAKING"] = 3] = "AGENT_STOPPED_SPEAKING";
+  VPAEvent2[VPAEvent2["USER_SPEECH_COMMITTED"] = 4] = "USER_SPEECH_COMMITTED";
+  VPAEvent2[VPAEvent2["AGENT_SPEECH_COMMITTED"] = 5] = "AGENT_SPEECH_COMMITTED";
+  VPAEvent2[VPAEvent2["AGENT_SPEECH_INTERRUPTED"] = 6] = "AGENT_SPEECH_INTERRUPTED";
+  VPAEvent2[VPAEvent2["FUNCTION_CALLS_COLLECTED"] = 7] = "FUNCTION_CALLS_COLLECTED";
+  VPAEvent2[VPAEvent2["FUNCTION_CALLS_FINISHED"] = 8] = "FUNCTION_CALLS_FINISHED";
+  return VPAEvent2;
+})(VPAEvent || {});
+class AgentCallContext {
+  #agent;
+  #llmStream;
+  #metadata = /* @__PURE__ */ new Map();
+  static #current;
+  constructor(agent, llmStream) {
+    this.#agent = agent;
+    this.#llmStream = llmStream;
+    AgentCallContext.#current = this;
+  }
+  static getCurrent() {
+    return AgentCallContext.#current;
+  }
+  get agent() {
+    return this.#agent;
+  }
+  storeMetadata(key, value) {
+    this.#metadata.set(key, value);
+  }
+  getMetadata(key, orDefault = void 0) {
+    return this.#metadata.get(key) || orDefault;
+  }
+  get llmStream() {
+    return this.#llmStream;
+  }
 }
 const defaultBeforeLLMCallback = (agent, chatCtx) => {
-    return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });
+  return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });
 };
-const defaultBeforeTTSCallback = (
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-_, text) => {
-    return text;
+const defaultBeforeTTSCallback = (_, text) => {
+  return text;
 };
 const defaultAgentTranscriptionOptions = {
-    userTranscription: true,
-    agentTranscription: true,
-    agentTranscriptionSpeech: 1,
-    sentenceTokenizer: new BasicSentenceTokenizer(),
-    wordTokenizer: new BasicWordTokenizer(false),
-    hyphenateWord: hyphenateWord,
+  userTranscription: true,
+  agentTranscription: true,
+  agentTranscriptionSpeech: 1,
+  sentenceTokenizer: new BasicSentenceTokenizer(),
+  wordTokenizer: new BasicWordTokenizer(false),
+  hyphenateWord
 };
 const defaultVPAOptions = {
-    chatCtx: new ChatContext(),
-    allowInterruptions: true,
-    interruptSpeechDuration: 50,
-    interruptMinWords: 0,
-    minEndpointingDelay: 500,
-    maxRecursiveFncCalls: 1,
-    preemptiveSynthesis: false,
-    beforeLLMCallback: defaultBeforeLLMCallback,
-    beforeTTSCallback: defaultBeforeTTSCallback,
-    transcription: defaultAgentTranscriptionOptions,
+  chatCtx: new ChatContext(),
+  allowInterruptions: true,
+  interruptSpeechDuration: 50,
+  interruptMinWords: 0,
+  minEndpointingDelay: 500,
+  maxRecursiveFncCalls: 1,
+  preemptiveSynthesis: false,
+  beforeLLMCallback: defaultBeforeLLMCallback,
+  beforeTTSCallback: defaultBeforeTTSCallback,
+  transcription: defaultAgentTranscriptionOptions
 };
-/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */
-export class VoicePipelineAgent extends EventEmitter {
-    /** Minimum time played for the user speech to be committed to the chat context. */
-    MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
-    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
-    #vad;
-    #stt;
-    #llm;
-    #tts;
-    #opts;
-    #humanInput;
-    #agentOutput;
-    #trackPublishedFut = new Future();
-    #pendingAgentReply;
-    #agentReplyTask;
-    #playingSpeech;
-    #transcribedText = '';
-    #transcribedInterimText = '';
-    #speechQueueOpen = new Future();
-    #speechQueue = new AsyncIterableQueue();
-    #lastEndOfSpeechTime;
-    #updateStateTask;
-    #started = false;
-    #room;
-    #participant = null;
-    #deferredValidation;
-    #logger = log();
-    #agentPublication;
-    constructor(
-    /** Voice Activity Detection instance. */
-    vad, 
-    /** Speech-to-Text instance. */
-    stt, 
-    /** Large Language Model instance. */
-    llm, 
-    /** Text-to-Speech instance. */
-    tts, 
-    /** Additional VoicePipelineAgent options. */
-    opts = defaultVPAOptions) {
-        super();
-        this.#opts = { ...defaultVPAOptions, ...opts };
-        if (!stt.capabilities.streaming) {
-            stt = new STTStreamAdapter(stt, vad);
-        }
-        if (!tts.capabilities.streaming) {
-            tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());
-        }
-        this.#vad = vad;
-        this.#stt = stt;
-        this.#llm = llm;
-        this.#tts = tts;
-        this.#deferredValidation = new DeferredReplyValidation(this.#validateReplyIfPossible.bind(this), this.#opts.minEndpointingDelay);
-    }
-    get fncCtx() {
-        return this.#opts.fncCtx;
-    }
-    set fncCtx(ctx) {
-        this.#opts.fncCtx = ctx;
-    }
-    get chatCtx() {
-        return this.#opts.chatCtx;
-    }
-    get llm() {
-        return this.#llm;
-    }
-    get tts() {
-        return this.#tts;
-    }
-    get stt() {
-        return this.#stt;
-    }
-    get vad() {
-        return this.#vad;
-    }
-    /** Start the voice assistant. */
-    start(
-    /** The room to connect to. */
-    room, 
-    /**
-     * The participant to listen to.
-     *
-     * @remarks
-     * Can be a participant or an identity.
-     * If omitted, the first participant in the room will be selected.
-     */
-    participant = null) {
-        if (this.#started) {
-            throw new Error('voice assistant already started');
-        }
-        room.on(RoomEvent.ParticipantConnected, (participant) => {
-            // automatically link to the first participant that connects, if not already linked
-            if (this.#participant) {
-                return;
-            }
-            this.#linkParticipant.call(this, participant.identity);
+class VoicePipelineAgent extends EventEmitter {
+  /** Minimum time played for the user speech to be committed to the chat context. */
+  MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
+  static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
+  #vad;
+  #stt;
+  #llm;
+  #tts;
+  #opts;
+  #humanInput;
+  #agentOutput;
+  #trackPublishedFut = new Future();
+  #pendingAgentReply;
+  #agentReplyTask;
+  #playingSpeech;
+  #transcribedText = "";
+  #transcribedInterimText = "";
+  #speechQueueOpen = new Future();
+  #speechQueue = new AsyncIterableQueue();
+  #lastEndOfSpeechTime;
+  #updateStateTask;
+  #started = false;
+  #room;
+  #participant = null;
+  #deferredValidation;
+  #logger = log();
+  #agentPublication;
+  constructor(vad, stt, llm, tts, opts = defaultVPAOptions) {
+    super();
+    this.#opts = { ...defaultVPAOptions, ...opts };
+    if (!stt.capabilities.streaming) {
+      stt = new STTStreamAdapter(stt, vad);
+    }
+    if (!tts.capabilities.streaming) {
+      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());
+    }
+    this.#vad = vad;
+    this.#stt = stt;
+    this.#llm = llm;
+    this.#tts = tts;
+    this.#deferredValidation = new DeferredReplyValidation(
+      this.#validateReplyIfPossible.bind(this),
+      this.#opts.minEndpointingDelay
+    );
+  }
+  get fncCtx() {
+    return this.#opts.fncCtx;
+  }
+  set fncCtx(ctx) {
+    this.#opts.fncCtx = ctx;
+  }
+  get chatCtx() {
+    return this.#opts.chatCtx;
+  }
+  get llm() {
+    return this.#llm;
+  }
+  get tts() {
+    return this.#tts;
+  }
+  get stt() {
+    return this.#stt;
+  }
+  get vad() {
+    return this.#vad;
+  }
+  /** Start the voice assistant. */
+  start(room, participant = null) {
+    if (this.#started) {
+      throw new Error("voice assistant already started");
+    }
+    room.on(RoomEvent.ParticipantConnected, (participant2) => {
+      if (this.#participant) {
+        return;
+      }
+      this.#linkParticipant.call(this, participant2.identity);
+    });
+    this.#room = room;
+    this.#participant = participant;
+    if (participant) {
+      if (typeof participant === "string") {
+        this.#linkParticipant(participant);
+      } else {
+        this.#linkParticipant(participant.identity);
+      }
+    }
+    this.#run();
+  }
+  /** Play a speech source through the voice assistant. */
+  async say(source, allowInterruptions = true, addToChatCtx = true) {
+    await this.#trackPublishedFut.await;
+    const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);
+    const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);
+    newHandle.initialize(source, synthesisHandle);
+    this.#addSpeechForPlayout(newHandle);
+  }
+  #updateState(state, delay = 0) {
+    const runTask = (delay2) => {
+      return new CancellablePromise(async (resolve, _, onCancel) => {
+        var _a, _b;
+        let cancelled = false;
+        onCancel(() => {
+          cancelled = true;
         });
-        this.#room = room;
-        this.#participant = participant;
-        if (participant) {
-            if (typeof participant === 'string') {
-                this.#linkParticipant(participant);
-            }
-            else {
-                this.#linkParticipant(participant.identity);
-            }
+        await new Promise((resolve2) => setTimeout(resolve2, delay2));
+        if ((_a = this.#room) == null ? void 0 : _a.isConnected) {
+          if (!cancelled) {
+            await ((_b = this.#room.localParticipant) == null ? void 0 : _b.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state }));
+          }
         }
-        this.#run();
-    }
-    /** Play a speech source through the voice assistant. */
-    async say(source, allowInterruptions = true, addToChatCtx = true) {
-        await this.#trackPublishedFut.await;
-        const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);
-        const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);
-        newHandle.initialize(source, synthesisHandle);
-        this.#addSpeechForPlayout(newHandle);
-    }
-    #updateState(state, delay = 0) {
-        const runTask = (delay) => {
-            return new CancellablePromise(async (resolve, _, onCancel) => {
-                let cancelled = false;
-                onCancel(() => {
-                    cancelled = true;
-                });
-                await new Promise((resolve) => setTimeout(resolve, delay));
-                if (this.#room?.isConnected) {
-                    if (!cancelled) {
-                        await this.#room.localParticipant?.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state });
-                    }
-                }
-                resolve();
-            });
-        };
-        if (this.#updateStateTask) {
-            this.#updateStateTask.cancel();
-        }
-        this.#updateStateTask = runTask(delay);
-    }
-    #linkParticipant(participantIdentity) {
-        if (!this.#room) {
-            this.#logger.error('Room is not set');
-            return;
+        resolve();
+      });
+    };
+    if (this.#updateStateTask) {
+      this.#updateStateTask.cancel();
+    }
+    this.#updateStateTask = runTask(delay);
+  }
+  #linkParticipant(participantIdentity) {
+    if (!this.#room) {
+      this.#logger.error("Room is not set");
+      return;
+    }
+    this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;
+    if (!this.#participant) {
+      this.#logger.error(`Participant with identity ${participantIdentity} not found`);
+      return;
+    }
+    this.#humanInput = new HumanInput(this.#room, this.#vad, this.#stt, this.#participant);
+    this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {
+      this.emit(0 /* USER_STARTED_SPEAKING */);
+      this.#deferredValidation.onHumanStartOfSpeech(event);
+    });
+    this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {
+      if (!this.#trackPublishedFut.done) {
+        return;
+      }
+      if (!this.#agentOutput) {
+        throw new Error("agent output is undefined");
+      }
+      let tv = 1;
+      if (this.#opts.allowInterruptions) {
+        tv = Math.max(0, 1 - event.probability);
+        this.#agentOutput.playout.targetVolume = tv;
+      }
+      if (event.speechDuration >= this.#opts.interruptSpeechDuration) {
+        this.#interruptIfPossible();
+      }
+    });
+    this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
+      this.emit(0 /* USER_STARTED_SPEAKING */);
+      this.#deferredValidation.onHumanEndOfSpeech(event);
+      this.#lastEndOfSpeechTime = Date.now();
+    });
+    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+      this.#transcribedInterimText = event.alternatives[0].text;
+    });
+    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
+      const newTranscript = event.alternatives[0].text;
+      if (!newTranscript) return;
+      this.#logger.child({ userTranscript: newTranscript }).debug("received user transcript");
+      this.#transcribedText += (this.#transcribedText ? " " : "") + newTranscript;
+      if (this.#opts.preemptiveSynthesis && (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
+        this.#synthesizeAgentReply();
+      }
+      this.#deferredValidation.onHumanFinalTranscript(newTranscript);
+      const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);
+      if (words.length >= 3) {
+        this.#interruptIfPossible();
+      }
+    });
+  }
+  async #run() {
+    var _a, _b;
+    this.#updateState("initializing");
+    const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);
+    const track = LocalAudioTrack.createAudioTrack("assistant_voice", audioSource);
+    this.#agentPublication = await ((_b = (_a = this.#room) == null ? void 0 : _a.localParticipant) == null ? void 0 : _b.publishTrack(
+      track,
+      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE })
+    ));
+    const agentPlayout = new AgentPlayout(audioSource);
+    this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);
+    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {
+      this.emit(2 /* AGENT_STARTED_SPEAKING */);
+      this.#updateState("speaking");
+    });
+    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {
+      this.emit(3 /* AGENT_STOPPED_SPEAKING */);
+      this.#updateState("listening");
+    });
+    this.#trackPublishedFut.resolve();
+    while (true) {
+      await this.#speechQueueOpen.await;
+      for await (const speech of this.#speechQueue) {
+        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;
+        this.#playingSpeech = speech;
+        await this.#playSpeech(speech);
+        this.#playingSpeech = void 0;
+      }
+      this.#speechQueueOpen = new Future();
+    }
+  }
+  #synthesizeAgentReply() {
+    var _a;
+    (_a = this.#pendingAgentReply) == null ? void 0 : _a.cancel();
+    if (this.#humanInput && this.#humanInput.speaking) {
+      this.#updateState("thinking", 200);
+    }
+    this.#pendingAgentReply = SpeechHandle.createAssistantReply(
+      this.#opts.allowInterruptions,
+      true,
+      this.#transcribedText
+    );
+    const newHandle = this.#pendingAgentReply;
+    this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
+  }
+  #synthesizeAnswerTask(oldTask, handle) {
+    return new CancellablePromise(async (resolve, _, onCancel) => {
+      let cancelled = false;
+      onCancel(() => {
+        cancelled = true;
+      });
+      if (oldTask) {
+        await gracefullyCancel(oldTask);
+      }
+      const copiedCtx = this.chatCtx.copy();
+      const playingSpeech = this.#playingSpeech;
+      if (playingSpeech && playingSpeech.initialized) {
+        if ((!playingSpeech.userQuestion || playingSpeech.userCommitted) && !playingSpeech.speechCommitted) {
+          copiedCtx.messages.push(
+            ChatMessage.create({
+              // TODO(nbsp): uhhh unsure where to get the played text here
+              // text: playingSpeech.synthesisHandle.(theres no ttsForwarder here)
+              role: ChatRole.ASSISTANT
+            })
+          );
         }
-        this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;
-        if (!this.#participant) {
-            this.#logger.error(`Participant with identity ${participantIdentity} not found`);
-            return;
-        }
-        this.#humanInput = new HumanInput(this.#room, this.#vad, this.#stt, this.#participant);
-        this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {
-            this.emit(VPAEvent.USER_STARTED_SPEAKING);
-            this.#deferredValidation.onHumanStartOfSpeech(event);
-        });
-        this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {
-            if (!this.#trackPublishedFut.done) {
-                return;
-            }
-            if (!this.#agentOutput) {
-                throw new Error('agent output is undefined');
-            }
-            let tv = 1;
-            if (this.#opts.allowInterruptions) {
-                tv = Math.max(0, 1 - event.probability);
-                this.#agentOutput.playout.targetVolume = tv;
-            }
-            if (event.speechDuration >= this.#opts.interruptSpeechDuration) {
-                this.#interruptIfPossible();
-            }
-        });
-        this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
-            this.emit(VPAEvent.USER_STARTED_SPEAKING);
-            this.#deferredValidation.onHumanEndOfSpeech(event);
-            this.#lastEndOfSpeechTime = Date.now();
-        });
-        this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
-            this.#transcribedInterimText = event.alternatives[0].text;
-        });
-        this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
-            const newTranscript = event.alternatives[0].text;
-            if (!newTranscript)
-                return;
-            this.#logger.child({ userTranscript: newTranscript }).debug('received user transcript');
-            this.#transcribedText += (this.#transcribedText ? ' ' : '') + newTranscript;
-            if (this.#opts.preemptiveSynthesis &&
-                (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
-                this.#synthesizeAgentReply();
-            }
-            this.#deferredValidation.onHumanFinalTranscript(newTranscript);
-            const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);
-            if (words.length >= 3) {
-                // VAD can sometimes not detect that the human is speaking.
-                // to make the interruption more reliable, we also interrupt on the final transcript.
-                this.#interruptIfPossible();
-            }
-        });
-    }
-    async #run() {
-        this.#updateState('initializing');
-        const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);
-        const track = LocalAudioTrack.createAudioTrack('assistant_voice', audioSource);
-        this.#agentPublication = await this.#room?.localParticipant?.publishTrack(track, new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }));
-        const agentPlayout = new AgentPlayout(audioSource);
-        this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);
-        agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {
-            this.emit(VPAEvent.AGENT_STARTED_SPEAKING);
-            this.#updateState('speaking');
-        });
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {
-            this.emit(VPAEvent.AGENT_STOPPED_SPEAKING);
-            this.#updateState('listening');
-        });
-        this.#trackPublishedFut.resolve();
-        while (true) {
-            await this.#speechQueueOpen.await;
-            for await (const speech of this.#speechQueue) {
-                if (speech === _a.FLUSH_SENTINEL)
-                    break;
-                this.#playingSpeech = speech;
-                await this.#playSpeech(speech);
-                this.#playingSpeech = undefined;
-            }
-            this.#speechQueueOpen = new Future();
+      }
+      copiedCtx.messages.push(
+        ChatMessage.create({
+          text: handle == null ? void 0 : handle.userQuestion,
+          role: ChatRole.USER
+        })
+      );
+      if (cancelled) resolve();
+      let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
+      if (llmStream === false) {
+        handle == null ? void 0 : handle.cancel();
+        return;
+      }
+      if (cancelled) resolve();
+      if (!(llmStream instanceof LLMStream)) {
+        llmStream = await defaultBeforeLLMCallback(this, copiedCtx);
+      }
+      if (handle.interrupted) {
+        return;
+      }
+      const synthesisHandle = this.#synthesizeAgentSpeech(handle.id, llmStream);
+      handle.initialize(llmStream, synthesisHandle);
+      const elapsed = !!this.#lastEndOfSpeechTime ? Math.round((Date.now() - this.#lastEndOfSpeechTime) * 1e3) / 1e3 : -1;
+      this.#logger.child({ speechId: handle.id, elapsed }).debug("synthesizing agent reply");
+      resolve();
+    });
+  }
+  async #playSpeech(handle) {
+    try {
+      await handle.waitForInitialization();
+    } catch {
+      return;
+    }
+    await this.#agentPublication.waitForSubscription();
+    const synthesisHandle = handle.synthesisHandle;
+    if (synthesisHandle.interrupted) return;
+    const userQuestion = handle.userQuestion;
+    const playHandle = synthesisHandle.play();
+    const joinFut = playHandle.join();
+    const commitUserQuestionIfNeeded = () => {
+      if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted) return;
+      const isUsingTools2 = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
+      if (handle.allowInterruptions && !isUsingTools2 && playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT && !joinFut.done) {
+        return;
+      }
+      this.#logger.child({ userTranscript: userQuestion }).debug("committed user transcript");
+      const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });
+      this.chatCtx.messages.push(userMsg);
+      this.emit(4 /* USER_SPEECH_COMMITTED */, userMsg);
+      this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+      handle.markUserCommitted();
+    };
+    commitUserQuestionIfNeeded();
+    while (!joinFut.done) {
+      await new Promise(async (resolve) => {
+        setTimeout(resolve, 500);
+        await joinFut.await;
+        resolve();
+      });
+      commitUserQuestionIfNeeded();
+      if (handle.interrupted) break;
+    }
+    commitUserQuestionIfNeeded();
+    let collectedText = "";
+    const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
+    const extraToolsMessages = [];
+    let interrupted = handle.interrupted;
+    if (isUsingTools && !interrupted) {
+      if (!userQuestion || !handle.userCommitted) {
+        throw new Error("user speech should have been committed before using tools");
+      }
+      const llmStream = handle.source;
+      let newFunctionCalls = llmStream.functionCalls;
+      for (let i = 0; i < this.#opts.maxRecursiveFncCalls; i++) {
+        this.emit(7 /* FUNCTION_CALLS_COLLECTED */, newFunctionCalls);
+        const calledFuncs = [];
+        for (const func of newFunctionCalls) {
+          const task = func.func.execute(func.params).then(
+            (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),
+            (error) => ({ name: func.name, toolCallId: func.toolCallId, error })
+          );
+          calledFuncs.push({ ...func, task });
+          this.#logger.child({ function: func.name, speechId: handle.id }).debug("executing AI function");
+          try {
+            await task;
+          } catch {
+            this.#logger.child({ function: func.name, speechId: handle.id }).error("error executing AI function");
+          }
         }
-    }
-    #synthesizeAgentReply() {
-        this.#pendingAgentReply?.cancel();
-        if (this.#humanInput && this.#humanInput.speaking) {
-            this.#updateState('thinking', 200);
+        const toolCallsInfo = [];
+        const toolCallsResults = [];
+        for (const fnc of calledFuncs) {
+          const task = await fnc.task;
+          if (!task || task.result === void 0) continue;
+          toolCallsInfo.push(fnc);
+          toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));
         }
-        this.#pendingAgentReply = SpeechHandle.createAssistantReply(this.#opts.allowInterruptions, true, this.#transcribedText);
-        const newHandle = this.#pendingAgentReply;
-        this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
-    }
-    #synthesizeAnswerTask(oldTask, handle) {
-        return new CancellablePromise(async (resolve, _, onCancel) => {
-            let cancelled = false;
-            onCancel(() => {
-                cancelled = true;
-            });
-            if (oldTask) {
-                await gracefullyCancel(oldTask);
-            }
-            const copiedCtx = this.chatCtx.copy();
-            const playingSpeech = this.#playingSpeech;
-            if (playingSpeech && playingSpeech.initialized) {
-                if ((!playingSpeech.userQuestion || playingSpeech.userCommitted) &&
-                    !playingSpeech.speechCommitted) {
-                    // the speech is playing but not committed yet,
-                    // add it to the chat context for this new reply synthesis
-                    copiedCtx.messages.push(ChatMessage.create({
-                        // TODO(nbsp): uhhh unsure where to get the played text here
-                        // text: playingSpeech.synthesisHandle.(theres no ttsForwarder here)
-                        role: ChatRole.ASSISTANT,
-                    }));
-                }
-            }
-            copiedCtx.messages.push(ChatMessage.create({
-                text: handle?.userQuestion,
-                role: ChatRole.USER,
-            }));
-            if (cancelled)
-                resolve();
-            let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
-            if (llmStream === false) {
-                handle?.cancel();
-                return;
-            }
-            if (cancelled)
-                resolve();
-            // fallback to default impl if no custom/user stream is returned
-            if (!(llmStream instanceof LLMStream)) {
-                llmStream = (await defaultBeforeLLMCallback(this, copiedCtx));
-            }
-            if (handle.interrupted) {
-                return;
-            }
-            const synthesisHandle = this.#synthesizeAgentSpeech(handle.id, llmStream);
-            handle.initialize(llmStream, synthesisHandle);
-            // TODO(theomonnom): find a more reliable way to get the elapsed time from the last EOS
-            // (VAD could not have detected any speech — maybe unlikely?)
-            const elapsed = !!this.#lastEndOfSpeechTime
-                ? Math.round((Date.now() - this.#lastEndOfSpeechTime) * 1000) / 1000
-                : -1;
-            this.#logger.child({ speechId: handle.id, elapsed }).debug('synthesizing agent reply');
-            resolve();
+        if (!toolCallsInfo.length) break;
+        extraToolsMessages.push(ChatMessage.createToolCalls(toolCallsInfo, collectedText));
+        extraToolsMessages.push(...toolCallsResults);
+        const chatCtx = handle.source.chatCtx.copy();
+        chatCtx.messages.push(...extraToolsMessages);
+        const answerLLMStream = this.llm.chat({
+          chatCtx,
+          fncCtx: this.fncCtx
         });
-    }
-    async #playSpeech(handle) {
-        try {
-            await handle.waitForInitialization();
-        }
-        catch {
-            return;
-        }
-        await this.#agentPublication.waitForSubscription();
-        const synthesisHandle = handle.synthesisHandle;
-        if (synthesisHandle.interrupted)
-            return;
-        const userQuestion = handle.userQuestion;
-        const playHandle = synthesisHandle.play();
-        const joinFut = playHandle.join();
-        const commitUserQuestionIfNeeded = () => {
-            if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted)
-                return;
-            const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
-            // make sure at least some speech was played before committing the user message
-            // since we try to validate as fast as possible it is possible the agent gets interrupted
-            // really quickly (barely audible), we don't want to mark this question as "answered".
-            if (handle.allowInterruptions &&
-                !isUsingTools &&
-                playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT &&
-                !joinFut.done) {
-                return;
-            }
-            this.#logger.child({ userTranscript: userQuestion }).debug('committed user transcript');
-            const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });
-            this.chatCtx.messages.push(userMsg);
-            this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);
-            this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
-            handle.markUserCommitted();
-        };
-        // wait for the playHandle to finish and check every 1s if user question should be committed
-        commitUserQuestionIfNeeded();
-        while (!joinFut.done) {
-            await new Promise(async (resolve) => {
-                setTimeout(resolve, 500);
-                await joinFut.await;
-                resolve();
-            });
-            commitUserQuestionIfNeeded();
-            if (handle.interrupted)
-                break;
-        }
-        commitUserQuestionIfNeeded();
-        // TODO(nbsp): what goes here
-        let collectedText = '';
-        const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
-        const extraToolsMessages = []; // additional messages from the functions to add to the context
-        let interrupted = handle.interrupted;
-        // if the answer is using tools, execute the functions and automatically generate
-        // a response to the user question from the returned values
-        if (isUsingTools && !interrupted) {
-            if (!userQuestion || !handle.userCommitted) {
-                throw new Error('user speech should have been committed before using tools');
-            }
-            const llmStream = handle.source;
-            let newFunctionCalls = llmStream.functionCalls;
-            for (let i = 0; i < this.#opts.maxRecursiveFncCalls; i++) {
-                this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);
-                const calledFuncs = [];
-                for (const func of newFunctionCalls) {
-                    const task = func.func.execute(func.params).then((result) => ({ name: func.name, toolCallId: func.toolCallId, result }), (error) => ({ name: func.name, toolCallId: func.toolCallId, error }));
-                    calledFuncs.push({ ...func, task });
-                    this.#logger
-                        .child({ function: func.name, speechId: handle.id })
-                        .debug('executing AI function');
-                    try {
-                        await task;
-                    }
-                    catch {
-                        this.#logger
-                            .child({ function: func.name, speechId: handle.id })
-                            .error('error executing AI function');
-                    }
-                }
-                const toolCallsInfo = [];
-                const toolCallsResults = [];
-                for (const fnc of calledFuncs) {
-                    // ignore the function calls that return void
-                    const task = await fnc.task;
-                    if (!task || task.result === undefined)
-                        continue;
-                    toolCallsInfo.push(fnc);
-                    toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));
-                }
-                if (!toolCallsInfo.length)
-                    break;
-                // generate an answer from the tool calls
-                extraToolsMessages.push(ChatMessage.createToolCalls(toolCallsInfo, collectedText));
-                extraToolsMessages.push(...toolCallsResults);
-                const chatCtx = handle.source.chatCtx.copy();
-                chatCtx.messages.push(...extraToolsMessages);
-                const answerLLMStream = this.llm.chat({
-                    chatCtx,
-                    fncCtx: this.fncCtx,
-                });
-                const answerSynthesis = this.#synthesizeAgentSpeech(handle.id, answerLLMStream);
-                // replace the synthesis handle with the new one to allow interruption
-                handle.synthesisHandle = answerSynthesis;
-                const playHandle = answerSynthesis.play();
-                await playHandle.join().await;
-                // TODO(nbsp): what text goes here
-                collectedText = '';
-                interrupted = answerSynthesis.interrupted;
-                newFunctionCalls = answerLLMStream.functionCalls;
-                this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);
-                if (!newFunctionCalls)
-                    break;
-            }
-            if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
-                this.chatCtx.messages.push(...extraToolsMessages);
-                if (interrupted) {
-                    collectedText + '…';
-                }
-                const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
-                this.chatCtx.messages.push(msg);
-                handle.markSpeechCommitted();
-                if (interrupted) {
-                    this.emit(VPAEvent.AGENT_SPEECH_INTERRUPTED, msg);
-                }
-                else {
-                    this.emit(VPAEvent.AGENT_SPEECH_COMMITTED, msg);
-                }
-                this.#logger
-                    .child({
-                    agentTranscript: collectedText,
-                    interrupted,
-                    speechId: handle.id,
-                })
-                    .debug('committed agent speech');
-            }
-        }
-    }
-    #synthesizeAgentSpeech(speechId, source) {
-        if (!this.#agentOutput) {
-            throw new Error('agent output should be initialized when ready');
-        }
-        if (source instanceof LLMStream) {
-            source = llmStreamToStringIterable(speechId, source);
-        }
-        const ogSource = source;
-        if (!(typeof source === 'string')) {
-            // TODO(nbsp): itertools.tee
-        }
-        const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);
-        if (!ttsSource) {
-            throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');
-        }
-        return this.#agentOutput.synthesize(speechId, ttsSource);
-    }
-    async #validateReplyIfPossible() {
-        if (this.#playingSpeech && this.#playingSpeech.allowInterruptions) {
-            this.#logger
-                .child({ speechId: this.#playingSpeech.id })
-                .debug('skipping validation, agent is speaking and does not allow interruptions');
-            return;
-        }
-        if (!this.#pendingAgentReply) {
-            if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
-                return;
-            }
-            this.#synthesizeAgentReply();
-        }
-        if (!this.#pendingAgentReply) {
-            throw new Error('pending agent reply is undefined');
-        }
-        // in some bad timimg, we could end up with two pushed agent replies inside the speech queue.
-        // so make sure we directly interrupt every reply when validating a new one
-        if (this.#speechQueueOpen.done) {
-            for await (const speech of this.#speechQueue) {
-                if (speech === _a.FLUSH_SENTINEL)
-                    break;
-                if (!speech.isReply)
-                    continue;
-                if (!speech.allowInterruptions)
-                    speech.interrupt();
-            }
-        }
-        this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');
-        this.#addSpeechForPlayout(this.#pendingAgentReply);
-        this.#pendingAgentReply = undefined;
-        this.#transcribedInterimText = '';
-    }
-    #interruptIfPossible() {
-        if (!this.#playingSpeech ||
-            !this.#playingSpeech.allowInterruptions ||
-            this.#playingSpeech.interrupted) {
-            return;
-        }
-        if (this.#opts.interruptMinWords !== 0) {
-            // check the final/interim transcribed text for the minimum word count
-            // to interrupt the agent speech
-            const interimWords = this.#opts.transcription.wordTokenizer.tokenize(this.#transcribedInterimText);
-            if (interimWords.length < this.#opts.interruptMinWords) {
-                return;
-            }
-        }
-        this.#playingSpeech.interrupt();
-    }
-    #addSpeechForPlayout(handle) {
-        this.#speechQueue.put(handle);
-        this.#speechQueue.put(_a.FLUSH_SENTINEL);
-        this.#speechQueueOpen.resolve();
-    }
-    /** Close the voice assistant. */
-    async close() {
-        if (!this.#started) {
-            return;
-        }
-        this.#room?.removeAllListeners(RoomEvent.ParticipantConnected);
-        // TODO(nbsp): await this.#deferredValidation.close()
-    }
+        const answerSynthesis = this.#synthesizeAgentSpeech(handle.id, answerLLMStream);
+        handle.synthesisHandle = answerSynthesis;
+        const playHandle2 = answerSynthesis.play();
+        await playHandle2.join().await;
+        collectedText = "";
+        interrupted = answerSynthesis.interrupted;
+        newFunctionCalls = answerLLMStream.functionCalls;
+        this.emit(8 /* FUNCTION_CALLS_FINISHED */, calledFuncs);
+        if (!newFunctionCalls) break;
+      }
+    }
+    if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
+      this.chatCtx.messages.push(...extraToolsMessages);
+      if (interrupted) {
+        collectedText + "\u2026";
+      }
+      const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
+      this.chatCtx.messages.push(msg);
+      handle.markSpeechCommitted();
+      if (interrupted) {
+        this.emit(6 /* AGENT_SPEECH_INTERRUPTED */, msg);
+      } else {
+        this.emit(5 /* AGENT_SPEECH_COMMITTED */, msg);
+      }
+      this.#logger.child({
+        agentTranscript: collectedText,
+        interrupted,
+        speechId: handle.id
+      }).debug("committed agent speech");
+    }
+  }
+  #synthesizeAgentSpeech(speechId, source) {
+    if (!this.#agentOutput) {
+      throw new Error("agent output should be initialized when ready");
+    }
+    if (source instanceof LLMStream) {
+      source = llmStreamToStringIterable(speechId, source);
+    }
+    const ogSource = source;
+    if (!(typeof source === "string")) {
+    }
+    const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);
+    if (!ttsSource) {
+      throw new Error("beforeTTSCallback must return string or AsyncIterable<string>");
+    }
+    return this.#agentOutput.synthesize(speechId, ttsSource);
+  }
+  async #validateReplyIfPossible() {
+    if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {
+      this.#logger.child({ speechId: this.#playingSpeech.id }).debug("skipping validation, agent is speaking and does not allow interruptions");
+      return;
+    }
+    if (!this.#pendingAgentReply) {
+      if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
+        return;
+      }
+      this.#synthesizeAgentReply();
+    }
+    if (!this.#pendingAgentReply) {
+      throw new Error("pending agent reply is undefined");
+    }
+    if (this.#speechQueueOpen.done) {
+      for await (const speech of this.#speechQueue) {
+        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;
+        if (!speech.isReply) continue;
+        if (speech.allowInterruptions) speech.interrupt();
+      }
+    }
+    this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug("validated agent reply");
+    this.#addSpeechForPlayout(this.#pendingAgentReply);
+    this.#pendingAgentReply = void 0;
+    this.#transcribedInterimText = "";
+  }
+  #interruptIfPossible() {
+    if (!this.#playingSpeech || !this.#playingSpeech.allowInterruptions || this.#playingSpeech.interrupted) {
+      return;
+    }
+    if (this.#opts.interruptMinWords !== 0) {
+      const interimWords = this.#opts.transcription.wordTokenizer.tokenize(
+        this.#transcribedInterimText
+      );
+      if (interimWords.length < this.#opts.interruptMinWords) {
+        return;
+      }
+    }
+    this.#playingSpeech.interrupt();
+  }
+  #addSpeechForPlayout(handle) {
+    this.#speechQueue.put(handle);
+    this.#speechQueue.put(VoicePipelineAgent.FLUSH_SENTINEL);
+    this.#speechQueueOpen.resolve();
+  }
+  /** Close the voice assistant. */
+  async close() {
+    var _a;
+    if (!this.#started) {
+      return;
+    }
+    (_a = this.#room) == null ? void 0 : _a.removeAllListeners(RoomEvent.ParticipantConnected);
+  }
 }
-_a = VoicePipelineAgent;
 async function* llmStreamToStringIterable(speechId, stream) {
-    const startTime = Date.now();
-    let firstFrame = true;
-    for await (const chunk of stream) {
-        const content = chunk.choices[0]?.delta.content;
-        if (!content)
-            continue;
-        if (firstFrame) {
-            firstFrame = false;
-            log()
-                .child({ speechId, elapsed: Math.round(Date.now() - startTime) })
-                .debug('received first LLM token');
-        }
-        yield content;
-    }
+  var _a;
+  const startTime = Date.now();
+  let firstFrame = true;
+  for await (const chunk of stream) {
+    const content = (_a = chunk.choices[0]) == null ? void 0 : _a.delta.content;
+    if (!content) continue;
+    if (firstFrame) {
+      firstFrame = false;
+      log().child({ speechId, elapsed: Math.round(Date.now() - startTime) }).debug("received first LLM token");
+    }
+    yield content;
+  }
 }
-/** This class is used to try to find the best time to validate the agent reply. */
 class DeferredReplyValidation {
-    // if the STT gives us punctuation, we can try to validate the reply faster.
-    PUNCTUATION = '.!?';
-    PUNCTUATION_REDUCE_FACTOR = 0.75;
-    LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech
-    #validateFunc;
-    #validatingPromise;
-    #validatingFuture = new Future();
-    #lastFinalTranscript = '';
-    #lastRecvEndOfSpeechTime = 0;
-    #speaking = false;
-    #endOfSpeechDelay;
-    #finalTranscriptDelay;
-    constructor(validateFunc, minEndpointingDelay) {
-        this.#validateFunc = validateFunc;
-        this.#endOfSpeechDelay = minEndpointingDelay;
-        this.#finalTranscriptDelay = minEndpointingDelay;
-    }
-    get validating() {
-        return !this.#validatingFuture.done;
-    }
-    onHumanFinalTranscript(transcript) {
-        this.#lastFinalTranscript = transcript.trim();
-        if (this.#speaking)
-            return;
-        const hasRecentEndOfSpeech = Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;
-        let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;
-        delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;
-        this.#run(delay);
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    onHumanStartOfSpeech(_) {
-        this.#speaking = true;
-        // TODO(nbsp):
-        // if (this.validating) {
-        //   this.#validatingPromise.cancel()
-        // }
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    onHumanEndOfSpeech(_) {
-        this.#speaking = false;
-        this.#lastRecvEndOfSpeechTime = Date.now();
-        if (this.#lastFinalTranscript) {
-            const delay = this.#endWithPunctuation()
-                ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR
-                : 1;
-            this.#run(delay);
-        }
-    }
-    // TODO(nbsp): aclose
-    #endWithPunctuation() {
-        return (this.#lastFinalTranscript.length > 0 &&
-            this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]));
-    }
-    #resetStates() {
-        this.#lastFinalTranscript = '';
-        this.#lastRecvEndOfSpeechTime = 0;
-    }
-    #run(delay) {
-        const runTask = async (delay) => {
-            await new Promise((resolve) => setTimeout(resolve, delay));
-            this.#resetStates();
-            await this.#validateFunc();
-        };
-        this.#validatingFuture = new Future();
-        this.#validatingPromise = runTask(delay);
-    }
+  // if the STT gives us punctuation, we can try to validate the reply faster.
+  PUNCTUATION = ".!?";
+  PUNCTUATION_REDUCE_FACTOR = 0.75;
+  LATE_TRANSCRIPT_TOLERANCE = 1.5;
+  // late compared to end of speech
+  #validateFunc;
+  #validatingPromise;
+  #validatingFuture = new Future();
+  #lastFinalTranscript = "";
+  #lastRecvEndOfSpeechTime = 0;
+  #speaking = false;
+  #endOfSpeechDelay;
+  #finalTranscriptDelay;
+  constructor(validateFunc, minEndpointingDelay) {
+    this.#validateFunc = validateFunc;
+    this.#endOfSpeechDelay = minEndpointingDelay;
+    this.#finalTranscriptDelay = minEndpointingDelay;
+  }
+  get validating() {
+    return !this.#validatingFuture.done;
+  }
+  onHumanFinalTranscript(transcript) {
+    this.#lastFinalTranscript = transcript.trim();
+    if (this.#speaking) return;
+    const hasRecentEndOfSpeech = Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;
+    let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;
+    delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;
+    this.#run(delay);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  onHumanStartOfSpeech(_) {
+    this.#speaking = true;
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  onHumanEndOfSpeech(_) {
+    this.#speaking = false;
+    this.#lastRecvEndOfSpeechTime = Date.now();
+    if (this.#lastFinalTranscript) {
+      const delay = this.#endWithPunctuation() ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR : 1;
+      this.#run(delay);
+    }
+  }
+  // TODO(nbsp): aclose
+  #endWithPunctuation() {
+    return this.#lastFinalTranscript.length > 0 && this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]);
+  }
+  #resetStates() {
+    this.#lastFinalTranscript = "";
+    this.#lastRecvEndOfSpeechTime = 0;
+  }
+  #run(delay) {
+    const runTask = async (delay2) => {
+      await new Promise((resolve) => setTimeout(resolve, delay2));
+      this.#resetStates();
+      await this.#validateFunc();
+    };
+    this.#validatingFuture = new Future();
+    this.#validatingPromise = runTask(delay);
+  }
 }
+export {
+  AGENT_STATE_ATTRIBUTE,
+  AgentCallContext,
+  VPAEvent,
+  VoicePipelineAgent
+};
 //# sourceMappingURL=pipeline_agent.js.map