@livekit/agents 0.7.8 → 1.0.0-next.0

package/dist/pipeline/pipeline_agent.js

@@ -1,826 +0,0 @@
-import {
-  AudioSource,
-  LocalAudioTrack,
-  RoomEvent,
-  TrackPublishOptions,
-  TrackSource
-} from "@livekit/rtc-node";
-import { randomUUID } from "node:crypto";
-import EventEmitter from "node:events";
-import {
-  ATTRIBUTE_TRANSCRIPTION_FINAL,
-  ATTRIBUTE_TRANSCRIPTION_TRACK_ID,
-  TOPIC_TRANSCRIPTION
-} from "../constants.js";
-import { ChatContext, ChatMessage, ChatRole, LLMEvent, LLMStream } from "../llm/index.js";
-import { log } from "../log.js";
-import { StreamAdapter as STTStreamAdapter, SpeechEventType } from "../stt/index.js";
-import {
-  SentenceTokenizer as BasicSentenceTokenizer,
-  WordTokenizer as BasicWordTokenizer,
-  hyphenateWord
-} from "../tokenize/basic/index.js";
-import { TextAudioSynchronizer, defaultTextSyncOptions } from "../transcription.js";
-import { TTSEvent, StreamAdapter as TTSStreamAdapter } from "../tts/index.js";
-import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from "../utils.js";
-import { VADEventType } from "../vad.js";
-import { AgentOutput } from "./agent_output.js";
-import { AgentPlayout, AgentPlayoutEvent } from "./agent_playout.js";
-import { HumanInput, HumanInputEvent } from "./human_input.js";
-import { SpeechHandle } from "./speech_handle.js";
-const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
-let lastSpeechData;
-let speechData;
-var VPAEvent = /* @__PURE__ */ ((VPAEvent2) => {
-  VPAEvent2[VPAEvent2["USER_STARTED_SPEAKING"] = 0] = "USER_STARTED_SPEAKING";
-  VPAEvent2[VPAEvent2["USER_STOPPED_SPEAKING"] = 1] = "USER_STOPPED_SPEAKING";
-  VPAEvent2[VPAEvent2["AGENT_STARTED_SPEAKING"] = 2] = "AGENT_STARTED_SPEAKING";
-  VPAEvent2[VPAEvent2["AGENT_STOPPED_SPEAKING"] = 3] = "AGENT_STOPPED_SPEAKING";
-  VPAEvent2[VPAEvent2["USER_SPEECH_COMMITTED"] = 4] = "USER_SPEECH_COMMITTED";
-  VPAEvent2[VPAEvent2["AGENT_SPEECH_COMMITTED"] = 5] = "AGENT_SPEECH_COMMITTED";
-  VPAEvent2[VPAEvent2["AGENT_SPEECH_INTERRUPTED"] = 6] = "AGENT_SPEECH_INTERRUPTED";
-  VPAEvent2[VPAEvent2["FUNCTION_CALLS_COLLECTED"] = 7] = "FUNCTION_CALLS_COLLECTED";
-  VPAEvent2[VPAEvent2["FUNCTION_CALLS_FINISHED"] = 8] = "FUNCTION_CALLS_FINISHED";
-  VPAEvent2[VPAEvent2["METRICS_COLLECTED"] = 9] = "METRICS_COLLECTED";
-  return VPAEvent2;
-})(VPAEvent || {});
-class AgentCallContext {
-  #agent;
-  #llmStream;
-  #metadata = /* @__PURE__ */ new Map();
-  #extraChatMessages = [];
-  static #current;
-  constructor(agent, llmStream) {
-    this.#agent = agent;
-    this.#llmStream = llmStream;
-    AgentCallContext.#current = this;
-  }
-  static getCurrent() {
-    return AgentCallContext.#current;
-  }
-  get agent() {
-    return this.#agent;
-  }
-  storeMetadata(key, value) {
-    this.#metadata.set(key, value);
-  }
-  getMetadata(key, orDefault = void 0) {
-    return this.#metadata.get(key) || orDefault;
-  }
-  get llmStream() {
-    return this.#llmStream;
-  }
-  get extraChatMessages() {
-    return this.#extraChatMessages;
-  }
-  addExtraChatMessage(message) {
-    this.#extraChatMessages.push(message);
-  }
-}
-const defaultBeforeLLMCallback = (agent, chatCtx) => {
-  return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });
-};
-const defaultBeforeTTSCallback = (_, text) => {
-  return text;
-};
-const defaultAgentTranscriptionOptions = {
-  userTranscription: true,
-  agentTranscription: true,
-  agentTranscriptionSpeech: 1,
-  sentenceTokenizer: new BasicSentenceTokenizer(),
-  wordTokenizer: new BasicWordTokenizer(false),
-  hyphenateWord
-};
-const defaultVPAOptions = {
-  chatCtx: new ChatContext(),
-  allowInterruptions: true,
-  interruptSpeechDuration: 50,
-  interruptMinWords: 0,
-  minEndpointingDelay: 500,
-  maxNestedFncCalls: 1,
-  preemptiveSynthesis: false,
-  beforeLLMCallback: defaultBeforeLLMCallback,
-  beforeTTSCallback: defaultBeforeTTSCallback,
-  transcription: defaultAgentTranscriptionOptions
-};
-class VoicePipelineAgent extends EventEmitter {
-  /** Minimum time played for the user speech to be committed to the chat context. */
-  MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
-  static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
-  #vad;
-  #stt;
-  #llm;
-  #tts;
-  #opts;
-  #humanInput;
-  #agentOutput;
-  #trackPublishedFut = new Future();
-  #pendingAgentReply;
-  #agentReplyTask;
-  #playingSpeech;
-  transcribedText = "";
-  #transcribedInterimText = "";
-  #speechQueueOpen = new Future();
-  #speechQueue = new AsyncIterableQueue();
-  #updateStateTask;
-  #started = false;
-  #room;
-  #participant = null;
-  #deferredValidation;
-  #logger = log();
-  #agentPublication;
-  #lastFinalTranscriptTime;
-  #lastSpeechTime;
-  #transcriptionId;
-  #agentTranscribedText = "";
-  constructor(vad, stt, llm, tts, opts = defaultVPAOptions) {
-    super();
-    this.#opts = { ...defaultVPAOptions, ...opts };
-    if (!stt.capabilities.streaming) {
-      stt = new STTStreamAdapter(stt, vad);
-    }
-    if (!tts.capabilities.streaming) {
-      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());
-    }
-    this.#vad = vad;
-    this.#stt = stt;
-    this.#llm = llm;
-    this.#tts = tts;
-    this.#deferredValidation = new DeferredReplyValidation(
-      this.#validateReplyIfPossible.bind(this),
-      this.#opts.minEndpointingDelay,
-      this,
-      this.#opts.turnDetector
-    );
-  }
-  get fncCtx() {
-    return this.#opts.fncCtx;
-  }
-  set fncCtx(ctx) {
-    this.#opts.fncCtx = ctx;
-  }
-  get chatCtx() {
-    return this.#opts.chatCtx;
-  }
-  get llm() {
-    return this.#llm;
-  }
-  get tts() {
-    return this.#tts;
-  }
-  get stt() {
-    return this.#stt;
-  }
-  get vad() {
-    return this.#vad;
-  }
-  /** Start the voice assistant. */
-  start(room, participant = null) {
-    if (this.#started) {
-      throw new Error("voice assistant already started");
-    }
-    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {
-      this.emit(9 /* METRICS_COLLECTED */, metrics);
-    });
-    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {
-      if (!speechData) return;
-      this.emit(9 /* METRICS_COLLECTED */, { ...metrics, sequenceId: speechData.sequenceId });
-    });
-    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {
-      const sequenceId = speechData ? speechData.sequenceId : lastSpeechData == null ? void 0 : lastSpeechData.sequenceId;
-      if (!sequenceId) return;
-      this.emit(9 /* METRICS_COLLECTED */, { ...metrics, sequenceId });
-    });
-    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {
-      this.emit(9 /* METRICS_COLLECTED */, metrics);
-    });
-    room.on(RoomEvent.ParticipantConnected, (participant2) => {
-      if (this.#participant) {
-        return;
-      }
-      this.#linkParticipant.call(this, participant2.identity);
-    });
-    this.#room = room;
-    this.#participant = participant;
-    if (participant) {
-      if (typeof participant === "string") {
-        this.#linkParticipant(participant);
-      } else {
-        this.#linkParticipant(participant.identity);
-      }
-    }
-    this.#run();
-  }
-  /** Play a speech source through the voice assistant. */
-  async say(source, allowInterruptions = true, addToChatCtx = true) {
-    await this.#trackPublishedFut.await;
-    let callContext;
-    let fncSource;
-    if (addToChatCtx) {
-      callContext = AgentCallContext.getCurrent();
-      if (source instanceof LLMStream) {
-        this.#logger.warn("LLMStream will be ignored for function call chat context");
-      } else if (typeof source === "string") {
-        fncSource = source;
-      } else {
-        fncSource = source;
-        source = new AsyncIterableQueue();
-      }
-    }
-    const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);
-    const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);
-    newHandle.initialize(source, synthesisHandle);
-    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {
-      this.#playingSpeech.addNestedSpeech(newHandle);
-    } else {
-      this.#addSpeechForPlayout(newHandle);
-    }
-    if (callContext && fncSource) {
-      let text;
-      if (typeof source === "string") {
-        text = fncSource;
-      } else {
-        text = "";
-        for await (const chunk of fncSource) {
-          source.put(chunk);
-          text += chunk;
-        }
-        source.close();
-      }
-      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));
-      this.#logger.child({ text }).debug("added speech to function call chat context");
-    }
-    return newHandle;
-  }
-  #updateState(state, delay = 0) {
-    const runTask = (delay2) => {
-      return new CancellablePromise(async (resolve, _, onCancel) => {
-        var _a, _b;
-        let cancelled = false;
-        onCancel(() => {
-          cancelled = true;
-        });
-        await new Promise((resolve2) => setTimeout(resolve2, delay2));
-        if ((_a = this.#room) == null ? void 0 : _a.isConnected) {
-          if (!cancelled) {
-            await ((_b = this.#room.localParticipant) == null ? void 0 : _b.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state }));
-          }
-        }
-        resolve();
-      });
-    };
-    if (this.#updateStateTask) {
-      this.#updateStateTask.cancel();
-    }
-    this.#updateStateTask = runTask(delay);
-  }
-  #linkParticipant(participantIdentity) {
-    if (!this.#room) {
-      this.#logger.error("Room is not set");
-      return;
-    }
-    this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;
-    if (!this.#participant) {
-      this.#logger.error(`Participant with identity ${participantIdentity} not found`);
-      return;
-    }
-    this.#humanInput = new HumanInput(
-      this.#room,
-      this.#vad,
-      this.#stt,
-      this.#participant,
-      this.#opts.noiseCancellation
-    );
-    this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {
-      this.emit(0 /* USER_STARTED_SPEAKING */);
-      this.#deferredValidation.onHumanStartOfSpeech(event);
-    });
-    this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {
-      if (!this.#trackPublishedFut.done) {
-        return;
-      }
-      if (!this.#agentOutput) {
-        throw new Error("agent output is undefined");
-      }
-      let tv = 1;
-      if (this.#opts.allowInterruptions) {
-        tv = Math.max(0, 1 - event.probability);
-        this.#agentOutput.playout.targetVolume = tv;
-      }
-      if (event.speechDuration >= this.#opts.interruptSpeechDuration) {
-        this.#interruptIfPossible();
-      }
-      if (event.rawAccumulatedSpeech > 0) {
-        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;
-      }
-    });
-    this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
-      this.emit(1 /* USER_STOPPED_SPEAKING */);
-      this.#deferredValidation.onHumanEndOfSpeech(event);
-    });
-    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, async (event) => {
-      if (!this.#transcriptionId) {
-        this.#transcriptionId = randomUUID();
-      }
-      this.#transcribedInterimText = event.alternatives[0].text;
-      await this.#publishTranscription(
-        this.#humanInput.participant.identity,
-        this.#humanInput.subscribedTrack.sid,
-        this.#transcribedInterimText,
-        false,
-        this.#transcriptionId
-      );
-    });
-    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, async (event) => {
-      const newTranscript = event.alternatives[0].text;
-      if (!newTranscript) return;
-      if (!this.#transcriptionId) {
-        this.#transcriptionId = randomUUID();
-      }
-      this.#lastFinalTranscriptTime = Date.now();
-      this.transcribedText += (this.transcribedText ? " " : "") + newTranscript;
-      await this.#publishTranscription(
-        this.#humanInput.participant.identity,
-        this.#humanInput.subscribedTrack.sid,
-        this.transcribedText,
-        true,
-        this.#transcriptionId
-      );
-      this.#transcriptionId = void 0;
-      if (this.#opts.preemptiveSynthesis && (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
-        this.#synthesizeAgentReply();
-      }
-      this.#deferredValidation.onHumanFinalTranscript(newTranscript);
-      const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);
-      if (words.length >= 3) {
-        this.#interruptIfPossible();
-      }
-    });
-  }
-  async #run() {
-    var _a, _b;
-    this.#updateState("initializing");
-    const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);
-    const track = LocalAudioTrack.createAudioTrack("assistant_voice", audioSource);
-    this.#agentPublication = await ((_b = (_a = this.#room) == null ? void 0 : _a.localParticipant) == null ? void 0 : _b.publishTrack(
-      track,
-      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE })
-    ));
-    const agentPlayout = new AgentPlayout(audioSource);
-    this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);
-    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {
-      this.emit(2 /* AGENT_STARTED_SPEAKING */);
-      this.#updateState("speaking");
-    });
-    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {
-      this.emit(3 /* AGENT_STOPPED_SPEAKING */);
-      this.#updateState("listening");
-    });
-    this.#trackPublishedFut.resolve();
-    while (true) {
-      await this.#speechQueueOpen.await;
-      for await (const speech of this.#speechQueue) {
-        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;
-        this.#playingSpeech = speech;
-        await this.#playSpeech(speech);
-        this.#playingSpeech = void 0;
-      }
-      this.#speechQueueOpen = new Future();
-    }
-  }
-  #synthesizeAgentReply() {
-    var _a;
-    (_a = this.#pendingAgentReply) == null ? void 0 : _a.cancel();
-    if (this.#humanInput && this.#humanInput.speaking) {
-      this.#updateState("thinking", 200);
-    }
-    this.#pendingAgentReply = SpeechHandle.createAssistantReply(
-      this.#opts.allowInterruptions,
-      true,
-      this.transcribedText
-    );
-    const newHandle = this.#pendingAgentReply;
-    this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
-  }
-  #synthesizeAnswerTask(oldTask, handle) {
-    return new CancellablePromise(async (resolve, _, onCancel) => {
-      let cancelled = false;
-      onCancel(() => {
-        cancelled = true;
-      });
-      if (oldTask) {
-        await gracefullyCancel(oldTask);
-      }
-      const copiedCtx = this.chatCtx.copy();
-      const playingSpeech = this.#playingSpeech;
-      if (playingSpeech && playingSpeech.initialized) {
-        if ((!playingSpeech.userQuestion || playingSpeech.userCommitted) && !playingSpeech.speechCommitted) {
-          copiedCtx.messages.push(
-            ChatMessage.create({
-              text: playingSpeech.synthesisHandle.text,
-              role: ChatRole.ASSISTANT
-            })
-          );
-        }
-      }
-      copiedCtx.messages.push(
-        ChatMessage.create({
-          text: handle == null ? void 0 : handle.userQuestion,
-          role: ChatRole.USER
-        })
-      );
-      speechData = { sequenceId: handle.id };
-      try {
-        if (cancelled) resolve();
-        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
-        if (llmStream === false) {
-          handle == null ? void 0 : handle.cancel();
-          return;
-        }
-        if (cancelled) resolve();
-        if (!(llmStream instanceof LLMStream)) {
-          llmStream = await defaultBeforeLLMCallback(this, copiedCtx);
-        }
-        if (handle.interrupted) {
-          return;
-        }
-        const synthesisHandle = this.#synthesizeAgentSpeech(handle.id, llmStream);
-        handle.initialize(llmStream, synthesisHandle);
-      } finally {
-        lastSpeechData = speechData;
-        speechData = void 0;
-      }
-      resolve();
-    });
-  }
-  async #playSpeech(handle) {
-    try {
-      await handle.waitForInitialization();
-    } catch {
-      return;
-    }
-    await this.#agentPublication.waitForSubscription();
-    const synthesisHandle = handle.synthesisHandle;
-    if (synthesisHandle.interrupted) return;
-    const userQuestion = handle.userQuestion;
-    const playHandle = synthesisHandle.play();
-    const joinFut = playHandle.join();
-    const commitUserQuestionIfNeeded = () => {
-      if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted) return;
-      const isUsingTools2 = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
-      if (handle.allowInterruptions && !isUsingTools2 && playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT && !joinFut.done) {
-        return;
-      }
-      this.#logger.child({ userTranscript: userQuestion }).debug("committed user transcript");
-      const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });
-      this.chatCtx.messages.push(userMsg);
-      this.emit(4 /* USER_SPEECH_COMMITTED */, userMsg);
-      this.transcribedText = this.transcribedText.slice(userQuestion.length);
-      handle.markUserCommitted();
-    };
-    commitUserQuestionIfNeeded();
-    while (!joinFut.done) {
-      await new Promise(async (resolve) => {
-        setTimeout(resolve, 500);
-        await joinFut.await;
-        resolve();
-      });
-      commitUserQuestionIfNeeded();
-      if (handle.interrupted) break;
-    }
-    commitUserQuestionIfNeeded();
-    let collectedText = this.#agentTranscribedText;
-    const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
-    const interrupted = handle.interrupted;
-    if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
-      if (handle.extraToolsMessages) {
-        this.chatCtx.messages.push(...handle.extraToolsMessages);
-      }
-      if (interrupted) {
-        collectedText += "\u2026";
-      }
-      const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
-      this.chatCtx.messages.push(msg);
-      handle.markSpeechCommitted();
-      if (interrupted) {
-        this.emit(6 /* AGENT_SPEECH_INTERRUPTED */, msg);
-      } else {
-        this.emit(5 /* AGENT_SPEECH_COMMITTED */, msg);
-      }
-      this.#logger.child({
-        agentTranscript: collectedText,
-        interrupted,
-        speechId: handle.id
-      }).debug("committed agent speech");
-      handle.setDone();
-    }
-    const executeFunctionCalls = async () => {
-      if (!isUsingTools || interrupted) return;
-      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {
-        this.#logger.child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth }).warn("max function calls nested depth reached");
-        return;
-      }
-      if (userQuestion && !handle.userCommitted) {
-        throw new Error("user speech should have been committed before using tools");
-      }
-      const llmStream = handle.source;
-      const newFunctionCalls = llmStream.functionCalls;
-      new AgentCallContext(this, llmStream);
-      this.emit(7 /* FUNCTION_CALLS_COLLECTED */, newFunctionCalls);
-      const calledFuncs = [];
-      for (const func of newFunctionCalls) {
-        const task2 = func.func.execute(func.params).then(
-          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),
-          (error) => ({ name: func.name, toolCallId: func.toolCallId, error })
-        );
-        calledFuncs.push({ ...func, task: task2 });
-        this.#logger.child({ function: func.name, speechId: handle.id }).debug("executing AI function");
-        try {
-          await task2;
-        } catch {
-          this.#logger.child({ function: func.name, speechId: handle.id }).error("error executing AI function");
-        }
-      }
-      const toolCallsInfo = [];
-      const toolCallsResults = [];
-      for (const fnc of calledFuncs) {
-        const task2 = await fnc.task;
-        if (!task2 || task2.result === void 0) continue;
-        toolCallsInfo.push(fnc);
-        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task2));
-      }
-      if (!toolCallsInfo.length) return;
-      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];
-      extraToolsMessages.push(...toolCallsResults);
-      const newSpeechHandle = SpeechHandle.createToolSpeech(
-        handle.allowInterruptions,
-        handle.addToChatCtx,
-        handle.fncNestedDepth + 1,
-        extraToolsMessages
-      );
-      const chatCtx = handle.source.chatCtx.copy();
-      chatCtx.messages.push(...extraToolsMessages);
-      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);
-      const answerLLMStream = this.llm.chat({
-        chatCtx,
-        fncCtx: this.fncCtx
-      });
-      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);
-      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);
-      handle.addNestedSpeech(newSpeechHandle);
-      this.emit(8 /* FUNCTION_CALLS_FINISHED */, calledFuncs);
-    };
-    let finished = false;
-    const task = executeFunctionCalls().then(() => {
-      finished = true;
-    });
-    while (!handle.nestedSpeechFinished) {
-      const changed = handle.nestedSpeechChanged();
-      await Promise.race([changed, task]);
-      while (handle.nestedSpeechHandles.length) {
-        const speech = handle.nestedSpeechHandles[0];
-        this.#playingSpeech = speech;
-        await this.#playSpeech(speech);
-        handle.nestedSpeechHandles.shift();
-        this.#playingSpeech = handle;
-      }
-      handle.nestedSpeechHandles.forEach(() => handle.nestedSpeechHandles.pop());
-      if (finished) {
-        handle.markNestedSpeechFinished();
-      }
-    }
-    handle.setDone();
-  }
-  async #publishTranscription(participantIdentity, trackSid, text, isFinal, id) {
-    this.#room.localParticipant.publishTranscription({
-      participantIdentity,
-      trackSid,
-      segments: [
-        {
-          text,
-          final: isFinal,
-          id,
-          startTime: BigInt(0),
-          endTime: BigInt(0),
-          language: ""
-        }
-      ]
-    });
-    const stream = await this.#room.localParticipant.streamText({
-      senderIdentity: participantIdentity,
-      topic: TOPIC_TRANSCRIPTION,
-      attributes: {
-        [ATTRIBUTE_TRANSCRIPTION_TRACK_ID]: trackSid,
-        [ATTRIBUTE_TRANSCRIPTION_FINAL]: isFinal.toString()
-      }
-    });
-    await stream.write(text);
-    await stream.close();
-  }
-  #synthesizeAgentSpeech(speechId, source) {
-    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
-    synchronizer.on("textUpdated", async (text) => {
-      var _a;
-      this.#agentTranscribedText = text.text;
-      await this.#publishTranscription(
-        this.#room.localParticipant.identity,
-        ((_a = this.#agentPublication) == null ? void 0 : _a.sid) ?? "",
-        text.text,
-        text.final,
-        text.id
-      );
-    });
-    if (!this.#agentOutput) {
-      throw new Error("agent output should be initialized when ready");
-    }
-    if (source instanceof LLMStream) {
-      source = llmStreamToStringIterable(speechId, source);
-    }
-    const ogSource = source;
-    if (!(typeof source === "string")) {
-    }
-    const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);
-    if (!ttsSource) {
-      throw new Error("beforeTTSCallback must return string or AsyncIterable<string>");
-    }
-    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);
-  }
-  async #validateReplyIfPossible() {
-    if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {
-      this.#logger.child({ speechId: this.#playingSpeech.id }).debug("skipping validation, agent is speaking and does not allow interruptions");
-      return;
-    }
-    if (!this.#pendingAgentReply) {
-      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {
-        return;
-      }
-      this.#synthesizeAgentReply();
-    }
-    if (!this.#pendingAgentReply) {
-      throw new Error("pending agent reply is undefined");
-    }
-    if (this.#speechQueueOpen.done) {
-      for await (const speech of this.#speechQueue) {
-        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;
-        if (!speech.isReply) continue;
-        if (speech.allowInterruptions) speech.interrupt();
-      }
-    }
-    this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug("validated agent reply");
-    if (this.#lastSpeechTime) {
-      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;
-      const transcriptionDelay = Math.max(
-        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,
-        0
-      );
-      const metrics = {
-        timestamp: Date.now(),
-        sequenceId: this.#pendingAgentReply.id,
-        endOfUtteranceDelay: timeSinceLastSpeech,
-        transcriptionDelay
-      };
-      this.emit(9 /* METRICS_COLLECTED */, metrics);
-    }
-    this.#addSpeechForPlayout(this.#pendingAgentReply);
-    this.#pendingAgentReply = void 0;
-    this.#transcribedInterimText = "";
-  }
-  #interruptIfPossible() {
-    if (!this.#playingSpeech || !this.#playingSpeech.allowInterruptions || this.#playingSpeech.interrupted) {
-      return;
-    }
-    if (this.#opts.interruptMinWords !== 0) {
-      const interimWords = this.#opts.transcription.wordTokenizer.tokenize(
-        this.#transcribedInterimText
-      );
-      if (interimWords.length < this.#opts.interruptMinWords) {
-        return;
-      }
-    }
-    this.#playingSpeech.interrupt();
-  }
-  #addSpeechForPlayout(handle) {
-    this.#speechQueue.put(handle);
-    this.#speechQueue.put(VoicePipelineAgent.FLUSH_SENTINEL);
-    this.#speechQueueOpen.resolve();
-  }
-  /** Close the voice assistant. */
-  async close() {
-    var _a;
-    if (!this.#started) {
-      return;
-    }
-    (_a = this.#room) == null ? void 0 : _a.removeAllListeners(RoomEvent.ParticipantConnected);
-  }
-}
-async function* llmStreamToStringIterable(speechId, stream) {
-  var _a;
-  const startTime = Date.now();
-  let firstFrame = true;
-  for await (const chunk of stream) {
-    const content = (_a = chunk.choices[0]) == null ? void 0 : _a.delta.content;
-    if (!content) continue;
-    if (firstFrame) {
-      firstFrame = false;
-      log().child({ speechId, elapsed: Math.round(Date.now() - startTime) }).debug("received first LLM token");
-    }
-    yield content;
-  }
-}
-class DeferredReplyValidation {
-  // if the STT gives us punctuation, we can try to validate the reply faster.
-  PUNCTUATION = ".!?";
-  PUNCTUATION_REDUCE_FACTOR = 0.75;
-  LATE_TRANSCRIPT_TOLERANCE = 1.5;
-  // late compared to end of speech
-  UNLIKELY_ENDPOINT_DELAY = 6e3;
-  #validateFunc;
-  #validatingPromise;
-  #validatingFuture = new Future();
-  #lastFinalTranscript = "";
-  #lastRecvEndOfSpeechTime = 0;
-  #speaking = false;
-  #endOfSpeechDelay;
-  #finalTranscriptDelay;
-  #turnDetector;
-  #agent;
-  #abort;
-  constructor(validateFunc, minEndpointingDelay, agent, turnDetector) {
-    this.#validateFunc = validateFunc;
-    this.#endOfSpeechDelay = minEndpointingDelay;
-    this.#finalTranscriptDelay = minEndpointingDelay;
-    this.#agent = agent;
-    this.#turnDetector = turnDetector;
-  }
-  get validating() {
-    return !this.#validatingFuture.done;
-  }
-  onHumanFinalTranscript(transcript) {
-    this.#lastFinalTranscript = transcript.trim();
-    if (this.#speaking) return;
-    const hasRecentEndOfSpeech = Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;
-    let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;
-    delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;
-    this.#run(delay);
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  onHumanStartOfSpeech(_) {
-    var _a;
-    this.#speaking = true;
-    if (this.validating) {
-      (_a = this.#abort) == null ? void 0 : _a.abort();
-    }
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  onHumanEndOfSpeech(_) {
-    this.#speaking = false;
-    this.#lastRecvEndOfSpeechTime = Date.now();
-    if (this.#lastFinalTranscript) {
-      const delay = this.#endWithPunctuation() ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR : 1e3;
-      this.#run(delay);
-    }
-  }
-  // TODO(nbsp): aclose
-  #endWithPunctuation() {
-    return this.#lastFinalTranscript.length > 0 && this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]);
-  }
-  #resetStates() {
-    this.#lastFinalTranscript = "";
-    this.#lastRecvEndOfSpeechTime = 0;
-  }
-  #run(delay) {
-    var _a;
-    const runTask = async (delay2, chatCtx, signal) => {
-      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {
-        const startTime = Date.now();
-        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);
-        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;
-        const elapsed = Date.now() - startTime;
-        if (eotProb < unlikelyThreshold) {
-          delay2 = this.UNLIKELY_ENDPOINT_DELAY;
-        }
-        delay2 = Math.max(0, delay2 - elapsed);
-      }
-      const timeout = setTimeout(() => {
-        this.#resetStates();
-        this.#validateFunc();
-      }, delay2);
-      signal.addEventListener("abort", () => {
-        clearTimeout(timeout);
-      });
-    };
-    (_a = this.#abort) == null ? void 0 : _a.abort();
-    this.#abort = new AbortController();
-    this.#validatingFuture = new Future();
-    const detectCtx = this.#agent.chatCtx.copy();
-    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });
-    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);
-  }
-}
-export {
-  AGENT_STATE_ATTRIBUTE,
-  AgentCallContext,
-  VPAEvent,
-  VoicePipelineAgent
-};
-//# sourceMappingURL=pipeline_agent.js.map