@livekit/agents 0.3.4 → 0.4.0

package/dist/pipeline/pipeline_agent.js

@@ -0,0 +1,661 @@
+var _a;
+import { AudioSource, LocalAudioTrack, RoomEvent, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
+import EventEmitter from 'node:events';
+import { LLMStream } from '../llm/index.js';
+import { ChatContext, ChatMessage, ChatRole } from '../llm/index.js';
+import { log } from '../log.js';
+import { SentenceTokenizer as BasicSentenceTokenizer, WordTokenizer as BasicWordTokenizer, hyphenateWord, } from '../tokenize/basic/index.js';
+import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+import { AgentOutput } from './agent_output.js';
+import { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';
+import { HumanInput, HumanInputEvent } from './human_input.js';
+import { SpeechHandle } from './speech_handle.js';
+export var VPAEvent;
+(function (VPAEvent) {
+    VPAEvent[VPAEvent["USER_STARTED_SPEAKING"] = 0] = "USER_STARTED_SPEAKING";
+    VPAEvent[VPAEvent["USER_STOPPED_SPEAKING"] = 1] = "USER_STOPPED_SPEAKING";
+    VPAEvent[VPAEvent["AGENT_STARTED_SPEAKING"] = 2] = "AGENT_STARTED_SPEAKING";
+    VPAEvent[VPAEvent["AGENT_STOPPED_SPEAKING"] = 3] = "AGENT_STOPPED_SPEAKING";
+    VPAEvent[VPAEvent["USER_SPEECH_COMMITTED"] = 4] = "USER_SPEECH_COMMITTED";
+    VPAEvent[VPAEvent["AGENT_SPEECH_COMMITTED"] = 5] = "AGENT_SPEECH_COMMITTED";
+    VPAEvent[VPAEvent["AGENT_SPEECH_INTERRUPTED"] = 6] = "AGENT_SPEECH_INTERRUPTED";
+    VPAEvent[VPAEvent["FUNCTION_CALLS_COLLECTED"] = 7] = "FUNCTION_CALLS_COLLECTED";
+    VPAEvent[VPAEvent["FUNCTION_CALLS_FINISHED"] = 8] = "FUNCTION_CALLS_FINISHED";
+})(VPAEvent || (VPAEvent = {}));
+export class AgentCallContext {
+    #agent;
+    #llmStream;
+    #metadata = new Map();
+    static #current;
+    constructor(agent, llmStream) {
+        this.#agent = agent;
+        this.#llmStream = llmStream;
+        AgentCallContext.#current = this;
+    }
+    static getCurrent() {
+        return AgentCallContext.#current;
+    }
+    get agent() {
+        return this.#agent;
+    }
+    storeMetadata(key, value) {
+        this.#metadata.set(key, value);
+    }
+    getMetadata(key, orDefault = undefined) {
+        return this.#metadata.get(key) || orDefault;
+    }
+    get llmStream() {
+        return this.#llmStream;
+    }
+}
+const defaultBeforeLLMCallback = (agent, chatCtx) => {
+    return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });
+};
+const defaultBeforeTTSCallback = (
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+_, text) => {
+    return text;
+};
+const defaultAgentTranscriptionOptions = {
+    userTranscription: true,
+    agentTranscription: true,
+    agentTranscriptionSpeech: 1,
+    sentenceTokenizer: new BasicSentenceTokenizer(),
+    wordTokenizer: new BasicWordTokenizer(false),
+    hyphenateWord: hyphenateWord,
+};
+const defaultVPAOptions = {
+    chatCtx: new ChatContext(),
+    allowInterruptions: true,
+    interruptSpeechDuration: 50,
+    interruptMinWords: 0,
+    minEndpointingDelay: 500,
+    maxRecursiveFncCalls: 1,
+    preemptiveSynthesis: false,
+    beforeLLMCallback: defaultBeforeLLMCallback,
+    beforeTTSCallback: defaultBeforeTTSCallback,
+    transcription: defaultAgentTranscriptionOptions,
+};
+/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */
+export class VoicePipelineAgent extends EventEmitter {
+    /** Minimum time played for the user speech to be committed to the chat context. */
+    MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
+    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+    #vad;
+    #stt;
+    #llm;
+    #tts;
+    #opts;
+    #humanInput;
+    #agentOutput;
+    #trackPublishedFut = new Future();
+    #pendingAgentReply;
+    #agentReplyTask;
+    #playingSpeech;
+    #transcribedText = '';
+    #transcribedInterimText = '';
+    #speechQueueOpen = new Future();
+    #speechQueue = new AsyncIterableQueue();
+    #lastEndOfSpeechTime;
+    #updateStateTask;
+    #started = false;
+    #room;
+    #participant = null;
+    #deferredValidation;
+    #logger = log();
+    #agentPublication;
+    constructor(
+    /** Voice Activity Detection instance. */
+    vad, 
+    /** Speech-to-Text instance. */
+    stt, 
+    /** Large Language Model instance. */
+    llm, 
+    /** Text-to-Speech instance. */
+    tts, 
+    /** Additional VoicePipelineAgent options. */
+    opts = defaultVPAOptions) {
+        super();
+        this.#opts = { ...defaultVPAOptions, ...opts };
+        this.#vad = vad;
+        this.#stt = stt;
+        this.#llm = llm;
+        this.#tts = tts;
+        this.#deferredValidation = new DeferredReplyValidation(this.#validateReplyIfPossible.bind(this), this.#opts.minEndpointingDelay);
+    }
+    get fncCtx() {
+        return this.#opts.fncCtx;
+    }
+    set fncCtx(ctx) {
+        this.#opts.fncCtx = ctx;
+    }
+    get chatCtx() {
+        return this.#opts.chatCtx;
+    }
+    get llm() {
+        return this.#llm;
+    }
+    get tts() {
+        return this.#tts;
+    }
+    get stt() {
+        return this.#stt;
+    }
+    get vad() {
+        return this.#vad;
+    }
+    /** Start the voice assistant. */
+    start(
+    /** The room to connect to. */
+    room, 
+    /**
+     * The participant to listen to.
+     *
+     * @remarks
+     * Can be a participant or an identity.
+     * If omitted, the first participant in the room will be selected.
+     */
+    participant = null) {
+        if (this.#started) {
+            throw new Error('voice assistant already started');
+        }
+        room.on(RoomEvent.ParticipantConnected, (participant) => {
+            // automatically link to the first participant that connects, if not already linked
+            if (this.#participant) {
+                return;
+            }
+            this.#linkParticipant.call(this, participant.identity);
+        });
+        this.#room = room;
+        this.#participant = participant;
+        if (participant) {
+            if (typeof participant === 'string') {
+                this.#linkParticipant(participant);
+            }
+            else {
+                this.#linkParticipant(participant.identity);
+            }
+        }
+        this.#run();
+    }
+    /** Play a speech source through the voice assistant. */
+    async say(source, allowInterruptions = true, addToChatCtx = true) {
+        await this.#trackPublishedFut.await;
+        const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);
+        const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);
+        newHandle.initialize(source, synthesisHandle);
+        this.#addSpeechForPlayout(newHandle);
+    }
+    #updateState(state, delay = 0) {
+        const runTask = (delay) => {
+            return new CancellablePromise(async (resolve, _, onCancel) => {
+                let cancelled = false;
+                onCancel(() => {
+                    cancelled = true;
+                });
+                await new Promise((resolve) => setTimeout(resolve, delay));
+                if (this.#room?.isConnected) {
+                    if (!cancelled) {
+                        await this.#room.localParticipant?.setAttributes({ ATTRIBUTE_AGENT_STATE: state });
+                    }
+                }
+                resolve();
+            });
+        };
+        if (this.#updateStateTask) {
+            this.#updateStateTask.cancel();
+        }
+        this.#updateStateTask = runTask(delay);
+    }
+    #linkParticipant(participantIdentity) {
+        if (!this.#room) {
+            this.#logger.error('Room is not set');
+            return;
+        }
+        this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;
+        if (!this.#participant) {
+            this.#logger.error(`Participant with identity ${participantIdentity} not found`);
+            return;
+        }
+        this.#humanInput = new HumanInput(this.#room, this.#vad, this.#stt, this.#participant);
+        this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {
+            this.emit(VPAEvent.USER_STARTED_SPEAKING);
+            this.#deferredValidation.onHumanStartOfSpeech(event);
+        });
+        this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {
+            if (!this.#trackPublishedFut.done) {
+                return;
+            }
+            if (!this.#agentOutput) {
+                throw new Error('agent output is undefined');
+            }
+            let tv = 1;
+            if (this.#opts.allowInterruptions) {
+                tv = Math.max(0, 1 - event.probability);
+                this.#agentOutput.playout.targetVolume = tv;
+            }
+            if (event.speechDuration >= this.#opts.interruptSpeechDuration) {
+                this.#interruptIfPossible();
+            }
+        });
+        this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
+            this.emit(VPAEvent.USER_STARTED_SPEAKING);
+            this.#deferredValidation.onHumanEndOfSpeech(event);
+            this.#lastEndOfSpeechTime = Date.now();
+        });
+        this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+            this.#transcribedInterimText = event.alternatives[0].text;
+        });
+        this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
+            const newTranscript = event.alternatives[0].text;
+            if (!newTranscript)
+                return;
+            this.#logger.child({ userTranscript: newTranscript }).debug('received user transcript');
+            this.#transcribedText += (this.#transcribedText ? ' ' : '') + newTranscript;
+            if (this.#opts.preemptiveSynthesis &&
+                (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
+                this.#synthesizeAgentReply();
+            }
+            this.#deferredValidation.onHumanFinalTranscript(newTranscript);
+            const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);
+            if (words.length >= 3) {
+                // VAD can sometimes not detect that the human is speaking.
+                // to make the interruption more reliable, we also interrupt on the final transcript.
+                this.#interruptIfPossible();
+            }
+        });
+    }
+    async #run() {
+        this.#updateState('initializing');
+        const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);
+        const track = LocalAudioTrack.createAudioTrack('assistant_voice', audioSource);
+        this.#agentPublication = await this.#room?.localParticipant?.publishTrack(track, new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }));
+        const agentPlayout = new AgentPlayout(audioSource);
+        this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);
+        agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {
+            this.emit(VPAEvent.AGENT_STARTED_SPEAKING);
+            this.#updateState('speaking');
+        });
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {
+            this.emit(VPAEvent.AGENT_STOPPED_SPEAKING);
+            this.#updateState('listening');
+        });
+        this.#trackPublishedFut.resolve();
+        while (true) {
+            await this.#speechQueueOpen.await;
+            for await (const speech of this.#speechQueue) {
+                if (speech === _a.FLUSH_SENTINEL)
+                    break;
+                this.#playingSpeech = speech;
+                await this.#playSpeech(speech);
+                this.#playingSpeech = undefined;
+            }
+            this.#speechQueueOpen = new Future();
+        }
+    }
+    #synthesizeAgentReply() {
+        this.#pendingAgentReply?.cancel();
+        if (this.#humanInput && this.#humanInput.speaking) {
+            this.#updateState('thinking', 200);
+        }
+        this.#pendingAgentReply = SpeechHandle.createAssistantReply(this.#opts.allowInterruptions, true, this.#transcribedText);
+        const newHandle = this.#pendingAgentReply;
+        this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
+    }
+    #synthesizeAnswerTask(oldTask, handle) {
+        return new CancellablePromise(async (resolve, _, onCancel) => {
+            let cancelled = false;
+            onCancel(() => {
+                cancelled = true;
+            });
+            if (oldTask) {
+                await gracefullyCancel(oldTask);
+            }
+            const copiedCtx = this.chatCtx.copy();
+            const playingSpeech = this.#playingSpeech;
+            if (playingSpeech && playingSpeech.initialized) {
+                if ((!playingSpeech.userQuestion || playingSpeech.userCommitted) &&
+                    !playingSpeech.speechCommitted) {
+                    // the speech is playing but not committed yet,
+                    // add it to the chat context for this new reply synthesis
+                    copiedCtx.messages.push(ChatMessage.create({
+                        // TODO(nbsp): uhhh unsure where to get the played text here
+                        // text: playingSpeech.synthesisHandle.(theres no ttsForwarder here)
+                        role: ChatRole.ASSISTANT,
+                    }));
+                }
+            }
+            copiedCtx.messages.push(ChatMessage.create({
+                text: handle?.userQuestion,
+                role: ChatRole.USER,
+            }));
+            if (cancelled)
+                resolve();
+            let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
+            if (llmStream === false) {
+                handle?.cancel();
+                return;
+            }
+            if (cancelled)
+                resolve();
+            // fallback to default impl if no custom/user stream is returned
+            if (!(llmStream instanceof LLMStream)) {
+                llmStream = (await defaultBeforeLLMCallback(this, copiedCtx));
+            }
+            if (handle.interrupted) {
+                return;
+            }
+            const synthesisHandle = this.#synthesizeAgentSpeech(handle.id, llmStream);
+            handle.initialize(llmStream, synthesisHandle);
+            // TODO(theomonnom): find a more reliable way to get the elapsed time from the last EOS
+            // (VAD could not have detected any speech — maybe unlikely?)
+            const elapsed = !!this.#lastEndOfSpeechTime
+                ? Math.round((Date.now() - this.#lastEndOfSpeechTime) * 1000) / 1000
+                : -1;
+            this.#logger.child({ speechId: handle.id, elapsed }).debug('synthesizing agent reply');
+            resolve();
+        });
+    }
+    async #playSpeech(handle) {
+        try {
+            await handle.waitForInitialization();
+        }
+        catch {
+            return;
+        }
+        await this.#agentPublication.waitForSubscription();
+        const synthesisHandle = handle.synthesisHandle;
+        if (synthesisHandle.interrupted)
+            return;
+        const userQuestion = handle.userQuestion;
+        const playHandle = synthesisHandle.play();
+        const joinFut = playHandle.join();
+        const commitUserQuestionIfNeeded = () => {
+            if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted)
+                return;
+            const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
+            // make sure at least some speech was played before committing the user message
+            // since we try to validate as fast as possible it is possible the agent gets interrupted
+            // really quickly (barely audible), we don't want to mark this question as "answered".
+            if (handle.allowInterruptions &&
+                !isUsingTools &&
+                playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT &&
+                !joinFut.done) {
+                return;
+            }
+            this.#logger.child({ userTranscript: userQuestion }).debug('committed user transcript');
+            const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });
+            this.chatCtx.messages.push(userMsg);
+            this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);
+            this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+        };
+        // wait for the playHandle to finish and check every 1s if user question should be committed
+        commitUserQuestionIfNeeded();
+        while (!joinFut.done) {
+            await new Promise(async (resolve) => {
+                setTimeout(resolve, 500);
+                await joinFut.await;
+                resolve();
+            });
+            commitUserQuestionIfNeeded();
+            if (handle.interrupted)
+                break;
+        }
+        commitUserQuestionIfNeeded();
+        // TODO(nbsp): what goes here
+        let collectedText = '';
+        const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
+        const extraToolsMessages = []; // additional messages from the functions to add to the context
+        let interrupted = handle.interrupted;
+        // if the answer is using tools, execute the functions and automatically generate
+        // a response to the user question from the returned values
+        if (isUsingTools && !interrupted) {
+            if (!userQuestion || handle.userCommitted) {
+                throw new Error('user speech should have been committed before using tools');
+            }
+            const llmStream = handle.source;
+            let newFunctionCalls = llmStream.functionCalls;
+            for (let i = 0; i < this.#opts.maxRecursiveFncCalls; i++) {
+                this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);
+                const calledFuncs = [];
+                for (const func of newFunctionCalls) {
+                    const task = func.func.execute(func.params).then((result) => ({ name: func.name, toolCallId: func.toolCallId, result }), (error) => ({ name: func.name, toolCallId: func.toolCallId, error }));
+                    calledFuncs.push({ ...func, task });
+                    this.#logger
+                        .child({ function: func.name, speechId: handle.id })
+                        .debug('executing AI function');
+                    try {
+                        await task;
+                    }
+                    catch {
+                        this.#logger
+                            .child({ function: func.name, speechId: handle.id })
+                            .error('error executing AI function');
+                    }
+                }
+                const toolCallsInfo = [];
+                const toolCallsResults = [];
+                for (const fnc of calledFuncs) {
+                    // ignore the function calls that return void
+                    const task = await fnc.task;
+                    if (!task || task.result === undefined)
+                        continue;
+                    toolCallsInfo.push(fnc);
+                    toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));
+                }
+                if (!toolCallsInfo.length)
+                    break;
+                // generate an answer from the tool calls
+                extraToolsMessages.push(ChatMessage.createToolCalls(toolCallsInfo, collectedText));
+                extraToolsMessages.push(...toolCallsResults);
+                const chatCtx = handle.source.chatCtx.copy();
+                chatCtx.messages.push(...extraToolsMessages);
+                const answerLLMStream = this.llm.chat({
+                    chatCtx,
+                    fncCtx: this.fncCtx,
+                });
+                const answerSynthesis = this.#synthesizeAgentSpeech(handle.id, answerLLMStream);
+                // replace the synthesis handle with the new one to allow interruption
+                handle.synthesisHandle = answerSynthesis;
+                const playHandle = answerSynthesis.play();
+                await playHandle.join().await;
+                // TODO(nbsp): what text goes here
+                collectedText = '';
+                interrupted = answerSynthesis.interrupted;
+                newFunctionCalls = answerLLMStream.functionCalls;
+                this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);
+                if (!newFunctionCalls)
+                    break;
+            }
+            if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
+                this.chatCtx.messages.push(...extraToolsMessages);
+                if (interrupted) {
+                    collectedText + '…';
+                }
+                const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
+                this.chatCtx.messages.push(msg);
+                handle.markSpeechCommitted();
+                if (interrupted) {
+                    this.emit(VPAEvent.AGENT_SPEECH_INTERRUPTED, msg);
+                }
+                else {
+                    this.emit(VPAEvent.AGENT_SPEECH_COMMITTED, msg);
+                }
+                this.#logger
+                    .child({
+                    agentTranscript: collectedText,
+                    interrupted,
+                    speechId: handle.id,
+                })
+                    .debug('committed agent speech');
+            }
+        }
+    }
+    #synthesizeAgentSpeech(speechId, source) {
+        if (!this.#agentOutput) {
+            throw new Error('agent output should be initialized when ready');
+        }
+        if (source instanceof LLMStream) {
+            source = llmStreamToStringIterable(speechId, source);
+        }
+        const ogSource = source;
+        if (!(typeof source === 'string')) {
+            // TODO(nbsp): itertools.tee
+        }
+        const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);
+        if (!ttsSource) {
+            throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');
+        }
+        return this.#agentOutput.synthesize(speechId, ttsSource);
+    }
+    async #validateReplyIfPossible() {
+        if (this.#playingSpeech && this.#playingSpeech.allowInterruptions) {
+            this.#logger
+                .child({ speechId: this.#playingSpeech.id })
+                .debug('skipping validation, agent is speaking and does not allow interruptions');
+            return;
+        }
+        if (!this.#pendingAgentReply) {
+            if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
+                return;
+            }
+            this.#synthesizeAgentReply();
+        }
+        if (!this.#pendingAgentReply) {
+            throw new Error('pending agent reply is undefined');
+        }
+        // in some bad timimg, we could end up with two pushed agent replies inside the speech queue.
+        // so make sure we directly interrupt every reply when validating a new one
+        if (this.#speechQueueOpen.done) {
+            for await (const speech of this.#speechQueue) {
+                if (speech === _a.FLUSH_SENTINEL)
+                    break;
+                if (!speech.isReply)
+                    continue;
+                if (!speech.allowInterruptions)
+                    speech.interrupt();
+            }
+        }
+        this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');
+        this.#addSpeechForPlayout(this.#pendingAgentReply);
+        this.#pendingAgentReply = undefined;
+        this.#transcribedInterimText = '';
+    }
+    #interruptIfPossible() {
+        if (!this.#playingSpeech ||
+            !this.#playingSpeech.allowInterruptions ||
+            this.#playingSpeech.interrupted) {
+            return;
+        }
+        if (this.#opts.interruptMinWords !== 0) {
+            // check the final/interim transcribed text for the minimum word count
+            // to interrupt the agent speech
+            const interimWords = this.#opts.transcription.wordTokenizer.tokenize(this.#transcribedInterimText);
+            if (interimWords.length < this.#opts.interruptMinWords) {
+                return;
+            }
+        }
+        this.#playingSpeech.interrupt();
+    }
+    #addSpeechForPlayout(handle) {
+        this.#speechQueue.put(handle);
+        this.#speechQueue.put(_a.FLUSH_SENTINEL);
+        this.#speechQueueOpen.resolve();
+    }
+    /** Close the voice assistant. */
+    async close() {
+        if (!this.#started) {
+            return;
+        }
+        this.#room?.removeAllListeners(RoomEvent.ParticipantConnected);
+        // TODO(nbsp): await this.#deferredValidation.close()
+    }
+}
+_a = VoicePipelineAgent;
+async function* llmStreamToStringIterable(speechId, stream) {
+    const startTime = Date.now();
+    let firstFrame = true;
+    for await (const chunk of stream) {
+        const content = chunk.choices[0].delta.content;
+        if (!content)
+            continue;
+        if (firstFrame) {
+            firstFrame = false;
+            log()
+                .child({ speechId, elapsed: Math.round(Date.now() * 1000 - startTime) / 1000 })
+                .debug('received first LLM token');
+        }
+        yield content;
+    }
+}
+/** This class is used to try to find the best time to validate the agent reply. */
+class DeferredReplyValidation {
+    // if the STT gives us punctuation, we can try to validate the reply faster.
+    PUNCTUATION = '.!?';
+    PUNCTUATION_REDUCE_FACTOR = 0.75;
+    LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech
+    #validateFunc;
+    #validatingPromise;
+    #validatingFuture = new Future();
+    #lastFinalTranscript = '';
+    #lastRecvEndOfSpeechTime = 0;
+    #speaking = false;
+    #endOfSpeechDelay;
+    #finalTranscriptDelay;
+    constructor(validateFunc, minEndpointingDelay) {
+        this.#validateFunc = validateFunc;
+        this.#endOfSpeechDelay = minEndpointingDelay;
+        this.#finalTranscriptDelay = minEndpointingDelay;
+    }
+    get validating() {
+        return !this.#validatingFuture.done;
+    }
+    onHumanFinalTranscript(transcript) {
+        this.#lastFinalTranscript = transcript.trim();
+        if (this.#speaking)
+            return;
+        const hasRecentEndOfSpeech = Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;
+        let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;
+        delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;
+        this.#run(delay);
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    onHumanStartOfSpeech(_) {
+        this.#speaking = true;
+        // TODO(nbsp):
+        // if (this.validating) {
+        //   this.#validatingPromise.cancel()
+        // }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    onHumanEndOfSpeech(_) {
+        this.#speaking = false;
+        this.#lastRecvEndOfSpeechTime = Date.now();
+        if (this.#lastFinalTranscript) {
+            const delay = this.#endWithPunctuation()
+                ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR
+                : 1;
+            this.#run(delay);
+        }
+    }
+    // TODO(nbsp): aclose
+    #endWithPunctuation() {
+        return (this.#lastFinalTranscript.length > 0 &&
+            this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]));
+    }
+    #resetStates() {
+        this.#lastFinalTranscript = '';
+        this.#lastRecvEndOfSpeechTime = 0;
+    }
+    #run(delay) {
+        const runTask = async (delay) => {
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            this.#resetStates();
+            await this.#validateFunc();
+        };
+        this.#validatingFuture = new Future();
+        this.#validatingPromise = runTask(delay);
+    }
+}
+//# sourceMappingURL=pipeline_agent.js.map