@livekit/agents 0.7.8 → 0.7.9

package/dist/pipeline/pipeline_agent.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/pipeline_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  NoiseCancellationOptions,\n  RemoteParticipant,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { randomUUID } from 'node:crypto';\nimport EventEmitter from 'node:events';\nimport {\n  ATTRIBUTE_TRANSCRIPTION_FINAL,\n  ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n  TOPIC_TRANSCRIPTION,\n} from '../constants.js';\nimport type {\n  CallableFunctionResult,\n  FunctionCallInfo,\n  FunctionContext,\n  LLM,\n} from '../llm/index.js';\nimport { ChatContext, ChatMessage, ChatRole, LLMEvent, LLMStream } from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { AgentMetrics, PipelineEOUMetrics } from '../metrics/base.js';\nimport { type STT, StreamAdapter as STTStreamAdapter, SpeechEventType } from '../stt/index.js';\nimport {\n  SentenceTokenizer as BasicSentenceTokenizer,\n  WordTokenizer as BasicWordTokenizer,\n  hyphenateWord,\n} from '../tokenize/basic/index.js';\nimport type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';\nimport { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';\nimport type { TTS } from '../tts/index.js';\nimport { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { SpeechSource, SynthesisHandle } from './agent_output.js';\nimport { AgentOutput } from './agent_output.js';\nimport { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';\nimport { HumanInput, HumanInputEvent } from './human_input.js';\nimport { SpeechHandle } from './speech_handle.js';\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\nlet lastSpeechData: { sequenceId: string } | undefined;\nlet speechData: { sequenceId: string } | undefined;\n\nexport type BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n) => LLMStream | false | void | Promise<LLMStream | false | void>;\n\nexport type BeforeTTSCallback = (\n  agent: VoicePipelineAgent,\n  source: string | AsyncIterable<string>,\n) => SpeechSource;\n\nexport enum VPAEvent {\n  USER_STARTED_SPEAKING,\n  USER_STOPPED_SPEAKING,\n  AGENT_STARTED_SPEAKING,\n  AGENT_STOPPED_SPEAKING,\n  USER_SPEECH_COMMITTED,\n  AGENT_SPEECH_COMMITTED,\n  AGENT_SPEECH_INTERRUPTED,\n  FUNCTION_CALLS_COLLECTED,\n  FUNCTION_CALLS_FINISHED,\n  METRICS_COLLECTED,\n}\n\nexport type VPACallbacks = {\n  [VPAEvent.USER_STARTED_SPEAKING]: () => void;\n  [VPAEvent.USER_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STARTED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.USER_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;\n  [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;\n  [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;\n  [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;\n};\n\ninterface TurnDetector {\n  unlikelyThreshold: number;\n  supportsLanguage: (language?: string) => boolean;\n  predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;\n}\n\nexport class AgentCallContext {\n  #agent: VoicePipelineAgent;\n  #llmStream: LLMStream;\n  #metadata = new Map<string, any>();\n  #extraChatMessages: ChatMessage[] = [];\n  static #current: AgentCallContext;\n\n  constructor(agent: VoicePipelineAgent, llmStream: LLMStream) {\n    this.#agent = agent;\n    this.#llmStream = llmStream;\n    AgentCallContext.#current = this;\n  }\n\n  static getCurrent(): AgentCallContext {\n    return AgentCallContext.#current;\n  }\n\n  get agent(): VoicePipelineAgent {\n    return this.#agent;\n  }\n\n  storeMetadata(key: string, value: any) {\n    this.#metadata.set(key, value);\n  }\n\n  getMetadata(key: string, orDefault: any = undefined) {\n    return this.#metadata.get(key) || orDefault;\n  }\n\n  get llmStream(): LLMStream {\n    return this.#llmStream;\n  }\n\n  get extraChatMessages() {\n    return this.#extraChatMessages;\n  }\n\n  addExtraChatMessage(message: ChatMessage) {\n    this.#extraChatMessages.push(message);\n  }\n}\n\nconst defaultBeforeLLMCallback: BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n): LLMStream => {\n  return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });\n};\n\nconst defaultBeforeTTSCallback: BeforeTTSCallback = (\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  _: VoicePipelineAgent,\n  text: string | AsyncIterable<string>,\n): string | AsyncIterable<string> => {\n  return text;\n};\n\nexport interface AgentTranscriptionOptions {\n  /** Whether to forward the user transcription to the client */\n  userTranscription: boolean;\n  /** Whether to forward the agent transcription to the client */\n  agentTranscription: boolean;\n  /**\n   * The speed at which the agent's speech transcription is forwarded to the client.\n   * We try to mimic the agent's speech speed by adjusting the transcription speed.\n   */\n  agentTranscriptionSpeech: number;\n  /**\n   * The tokenizer used to split the speech into sentences.\n   * This is used to decide when to mark a transcript as final for the agent transcription.\n   */\n  sentenceTokenizer: SentenceTokenizer;\n  /**\n   * The tokenizer used to split the speech into words.\n   * This is used to simulate the \"interim results\" of the agent transcription.\n   */\n  wordTokenizer: WordTokenizer;\n  /**\n   * A function that takes a string (word) as input and returns a list of strings,\n   * representing the hyphenated parts of the word.\n   */\n  hyphenateWord: (word: string) => string[];\n}\n\nconst defaultAgentTranscriptionOptions: AgentTranscriptionOptions = {\n  userTranscription: true,\n  agentTranscription: true,\n  agentTranscriptionSpeech: 1,\n  sentenceTokenizer: new BasicSentenceTokenizer(),\n  wordTokenizer: new BasicWordTokenizer(false),\n  hyphenateWord: hyphenateWord,\n};\n\nexport interface VPAOptions {\n  /** Chat context for the assistant. */\n  chatCtx?: ChatContext;\n  /** Function context for the assistant. */\n  fncCtx?: FunctionContext;\n  /** Whether to allow the user to interrupt the assistant. */\n  allowInterruptions: boolean;\n  /** Minimum duration of speech to consider for interruption. */\n  interruptSpeechDuration: number;\n  /** Minimum number of words to consider for interuption. This may increase latency. */\n  interruptMinWords: number;\n  /** Delay to wait before considering the user speech done. */\n  minEndpointingDelay: number;\n  maxNestedFncCalls: number;\n  /* Whether to preemptively synthesize responses. */\n  preemptiveSynthesis: boolean;\n  /*\n   * Callback called when the assistant is about to synthesize a reply.\n   *\n   * @remarks\n   * Returning void will create a default LLM stream.\n   * You can also return your own LLM stream by calling `llm.chat()`.\n   * Returning `false` ill cancel the synthesis of the reply.\n   */\n  beforeLLMCallback: BeforeLLMCallback;\n  /*\n   * Callback called when the assistant is about to synthesize speech.\n   *\n   * @remarks\n   * This can be used to customize text before synthesis\n   * (e.g. editing the pronunciation of a word).\n   */\n  beforeTTSCallback: BeforeTTSCallback;\n  /** Options for assistant transcription. */\n  transcription: AgentTranscriptionOptions;\n  /** Turn detection model to use. */\n  turnDetector?: TurnDetector;\n  /** Noise cancellation options. */\n  noiseCancellation?: NoiseCancellationOptions;\n}\n\nconst defaultVPAOptions: VPAOptions = {\n  chatCtx: new ChatContext(),\n  allowInterruptions: true,\n  interruptSpeechDuration: 50,\n  interruptMinWords: 0,\n  minEndpointingDelay: 500,\n  maxNestedFncCalls: 1,\n  preemptiveSynthesis: false,\n  beforeLLMCallback: defaultBeforeLLMCallback,\n  beforeTTSCallback: defaultBeforeTTSCallback,\n  transcription: defaultAgentTranscriptionOptions,\n};\n\n/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */\nexport class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<VPACallbacks>) {\n  /** Minimum time played for the user speech to be committed to the chat context. */\n  readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n\n  #vad: VAD;\n  #stt: STT;\n  #llm: LLM;\n  #tts: TTS;\n  #opts: VPAOptions;\n  #humanInput?: HumanInput;\n  #agentOutput?: AgentOutput;\n  #trackPublishedFut = new Future();\n  #pendingAgentReply?: SpeechHandle;\n  #agentReplyTask?: CancellablePromise<void>;\n  #playingSpeech?: SpeechHandle;\n  transcribedText = '';\n  #transcribedInterimText = '';\n  #speechQueueOpen = new Future();\n  #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();\n  #updateStateTask?: CancellablePromise<void>;\n  #started = false;\n  #room?: Room;\n  #participant: RemoteParticipant | string | null = null;\n  #deferredValidation: DeferredReplyValidation;\n  #logger = log();\n  #agentPublication?: LocalTrackPublication;\n  #lastFinalTranscriptTime?: number;\n  #lastSpeechTime?: number;\n  #transcriptionId?: string;\n  #agentTranscribedText = '';\n\n  constructor(\n    /** Voice Activity Detection instance. */\n    vad: VAD,\n    /** Speech-to-Text instance. */\n    stt: STT,\n    /** Large Language Model instance. */\n    llm: LLM,\n    /** Text-to-Speech instance. */\n    tts: TTS,\n    /** Additional VoicePipelineAgent options. */\n    opts: Partial<VPAOptions> = defaultVPAOptions,\n  ) {\n    super();\n\n    this.#opts = { ...defaultVPAOptions, ...opts };\n\n    if (!stt.capabilities.streaming) {\n      stt = new STTStreamAdapter(stt, vad);\n    }\n\n    if (!tts.capabilities.streaming) {\n      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());\n    }\n\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#llm = llm;\n    this.#tts = tts;\n\n    this.#deferredValidation = new DeferredReplyValidation(\n      this.#validateReplyIfPossible.bind(this),\n      this.#opts.minEndpointingDelay,\n      this,\n      this.#opts.turnDetector,\n    );\n  }\n\n  get fncCtx(): FunctionContext | undefined {\n    return this.#opts.fncCtx;\n  }\n\n  set fncCtx(ctx: FunctionContext) {\n    this.#opts.fncCtx = ctx;\n  }\n\n  get chatCtx(): ChatContext {\n    return this.#opts.chatCtx!;\n  }\n\n  get llm(): LLM {\n    return this.#llm;\n  }\n\n  get tts(): TTS {\n    return this.#tts;\n  }\n\n  get stt(): STT {\n    return this.#stt;\n  }\n\n  get vad(): VAD {\n    return this.#vad;\n  }\n\n  /** Start the voice assistant. */\n  start(\n    /** The room to connect to. */\n    room: Room,\n    /**\n     * The participant to listen to.\n     *\n     * @remarks\n     * Can be a participant or an identity.\n     * If omitted, the first participant in the room will be selected.\n     */\n    participant: RemoteParticipant | string | null = null,\n  ) {\n    if (this.#started) {\n      throw new Error('voice assistant already started');\n    }\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {\n      if (!speechData) return;\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });\n    });\n\n    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {\n      const sequenceId = speechData ? speechData.sequenceId : lastSpeechData?.sequenceId;\n      if (!sequenceId) return;\n\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId });\n    });\n\n    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n      // automatically link to the first participant that connects, if not already linked\n      if (this.#participant) {\n        return;\n      }\n      this.#linkParticipant.call(this, participant.identity!);\n    });\n\n    this.#room = room;\n    this.#participant = participant;\n\n    if (participant) {\n      if (typeof participant === 'string') {\n        this.#linkParticipant(participant);\n      } else {\n        this.#linkParticipant(participant.identity!);\n      }\n    }\n\n    this.#run();\n  }\n\n  /** Play a speech source through the voice assistant. */\n  async say(\n    source: string | LLMStream | AsyncIterable<string>,\n    allowInterruptions = true,\n    addToChatCtx = true,\n  ): Promise<SpeechHandle> {\n    await this.#trackPublishedFut.await;\n\n    let callContext: AgentCallContext | undefined;\n    let fncSource: string | AsyncIterable<string> | undefined;\n    if (addToChatCtx) {\n      callContext = AgentCallContext.getCurrent();\n      if (source instanceof LLMStream) {\n        this.#logger.warn('LLMStream will be ignored for function call chat context');\n      } else if (typeof source === 'string') {\n        fncSource = source;\n      } else {\n        fncSource = source;\n        source = new AsyncIterableQueue<string>();\n      }\n    }\n\n    const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);\n    const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);\n    newHandle.initialize(source, synthesisHandle);\n\n    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {\n      this.#playingSpeech.addNestedSpeech(newHandle);\n    } else {\n      this.#addSpeechForPlayout(newHandle);\n    }\n\n    if (callContext && fncSource) {\n      let text: string;\n      if (typeof source === 'string') {\n        text = fncSource as string;\n      } else {\n        text = '';\n        for await (const chunk of fncSource) {\n          (source as AsyncIterableQueue<string>).put(chunk);\n          text += chunk;\n        }\n        (source as AsyncIterableQueue<string>).close();\n      }\n\n      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));\n      this.#logger.child({ text }).debug('added speech to function call chat context');\n    }\n\n    return newHandle;\n  }\n\n  #updateState(state: AgentState, delay = 0) {\n    const runTask = (delay: number): CancellablePromise<void> => {\n      return new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n        await new Promise((resolve) => setTimeout(resolve, delay));\n        if (this.#room?.isConnected) {\n          if (!cancelled) {\n            await this.#room.localParticipant?.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state });\n          }\n        }\n        resolve();\n      });\n    };\n\n    if (this.#updateStateTask) {\n      this.#updateStateTask.cancel();\n    }\n\n    this.#updateStateTask = runTask(delay);\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.#room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.#participant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    this.#humanInput = new HumanInput(\n      this.#room,\n      this.#vad,\n      this.#stt,\n      this.#participant,\n      this.#opts.noiseCancellation,\n    );\n    this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STARTED_SPEAKING);\n      this.#deferredValidation.onHumanStartOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {\n      if (!this.#trackPublishedFut.done) {\n        return;\n      }\n      if (!this.#agentOutput) {\n        throw new Error('agent output is undefined');\n      }\n\n      let tv = 1;\n      if (this.#opts.allowInterruptions) {\n        tv = Math.max(0, 1 - event.probability);\n        this.#agentOutput.playout.targetVolume = tv;\n      }\n\n      if (event.speechDuration >= this.#opts.interruptSpeechDuration) {\n        this.#interruptIfPossible();\n      }\n\n      if (event.rawAccumulatedSpeech > 0) {\n        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;\n      }\n    });\n    this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STOPPED_SPEAKING);\n      this.#deferredValidation.onHumanEndOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, async (event) => {\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n      this.#transcribedInterimText = event.alternatives![0].text;\n\n      await this.#publishTranscription(\n        this.#humanInput!.participant.identity,\n        this.#humanInput!.subscribedTrack!.sid!,\n        this.#transcribedInterimText,\n        false,\n        this.#transcriptionId,\n      );\n    });\n    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, async (event) => {\n      const newTranscript = event.alternatives![0].text;\n      if (!newTranscript) return;\n\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n\n      this.#lastFinalTranscriptTime = Date.now();\n      this.transcribedText += (this.transcribedText ? ' ' : '') + newTranscript;\n\n      await this.#publishTranscription(\n        this.#humanInput!.participant.identity,\n        this.#humanInput!.subscribedTrack!.sid!,\n        this.transcribedText,\n        true,\n        this.#transcriptionId,\n      );\n\n      this.#transcriptionId = undefined;\n\n      if (\n        this.#opts.preemptiveSynthesis &&\n        (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)\n      ) {\n        this.#synthesizeAgentReply();\n      }\n\n      this.#deferredValidation.onHumanFinalTranscript(newTranscript);\n\n      const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);\n      if (words.length >= 3) {\n        // VAD can sometimes not detect that the human is speaking.\n        // to make the interruption more reliable, we also interrupt on the final transcript.\n        this.#interruptIfPossible();\n      }\n    });\n  }\n\n  async #run() {\n    this.#updateState('initializing');\n    const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);\n    const track = LocalAudioTrack.createAudioTrack('assistant_voice', audioSource);\n    this.#agentPublication = await this.#room?.localParticipant?.publishTrack(\n      track,\n      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n    );\n\n    const agentPlayout = new AgentPlayout(audioSource);\n    this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);\n\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {\n      this.emit(VPAEvent.AGENT_STARTED_SPEAKING);\n      this.#updateState('speaking');\n    });\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {\n      this.emit(VPAEvent.AGENT_STOPPED_SPEAKING);\n      this.#updateState('listening');\n    });\n\n    this.#trackPublishedFut.resolve();\n\n    while (true) {\n      await this.#speechQueueOpen.await;\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        this.#playingSpeech = undefined;\n      }\n      this.#speechQueueOpen = new Future();\n    }\n  }\n\n  #synthesizeAgentReply() {\n    this.#pendingAgentReply?.cancel();\n    if (this.#humanInput && this.#humanInput.speaking) {\n      this.#updateState('thinking', 200);\n    }\n\n    this.#pendingAgentReply = SpeechHandle.createAssistantReply(\n      this.#opts.allowInterruptions,\n      true,\n      this.transcribedText,\n    );\n    const newHandle = this.#pendingAgentReply;\n    this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);\n  }\n\n  #synthesizeAnswerTask(\n    oldTask: CancellablePromise<void> | undefined,\n    handle?: SpeechHandle,\n  ): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      const copiedCtx = this.chatCtx.copy();\n      const playingSpeech = this.#playingSpeech;\n      if (playingSpeech && playingSpeech.initialized) {\n        if (\n          (!playingSpeech.userQuestion || playingSpeech.userCommitted) &&\n          !playingSpeech.speechCommitted\n        ) {\n          // the speech is playing but not committed yet,\n          // add it to the chat context for this new reply synthesis\n          copiedCtx.messages.push(\n            ChatMessage.create({\n              text: playingSpeech.synthesisHandle.text,\n              role: ChatRole.ASSISTANT,\n            }),\n          );\n        }\n      }\n\n      copiedCtx.messages.push(\n        ChatMessage.create({\n          text: handle?.userQuestion,\n          role: ChatRole.USER,\n        }),\n      );\n\n      speechData = { sequenceId: handle!.id };\n\n      try {\n        if (cancelled) resolve();\n        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);\n        if (llmStream === false) {\n          handle?.cancel();\n          return;\n        }\n\n        if (cancelled) resolve();\n        // fallback to default impl if no custom/user stream is returned\n        if (!(llmStream instanceof LLMStream)) {\n          llmStream = (await defaultBeforeLLMCallback(this, copiedCtx)) as LLMStream;\n        }\n\n        if (handle!.interrupted) {\n          return;\n        }\n\n        const synthesisHandle = this.#synthesizeAgentSpeech(handle!.id, llmStream);\n        handle!.initialize(llmStream, synthesisHandle);\n      } finally {\n        lastSpeechData = speechData;\n        speechData = undefined;\n      }\n      resolve();\n    });\n  }\n\n  async #playSpeech(handle: SpeechHandle) {\n    try {\n      await handle.waitForInitialization();\n    } catch {\n      return;\n    }\n    await this.#agentPublication!.waitForSubscription();\n    const synthesisHandle = handle.synthesisHandle;\n    if (synthesisHandle.interrupted) return;\n\n    const userQuestion = handle.userQuestion;\n    const playHandle = synthesisHandle.play();\n    const joinFut = playHandle.join();\n\n    const commitUserQuestionIfNeeded = () => {\n      if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted) return;\n      const isUsingTools =\n        handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n\n      // make sure at least some speech was played before committing the user message\n      // since we try to validate as fast as possible it is possible the agent gets interrupted\n      // really quickly (barely audible), we don't want to mark this question as \"answered\".\n      if (\n        handle.allowInterruptions &&\n        !isUsingTools &&\n        playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT &&\n        !joinFut.done\n      ) {\n        return;\n      }\n\n      this.#logger.child({ userTranscript: userQuestion }).debug('committed user transcript');\n      const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });\n      this.chatCtx.messages.push(userMsg);\n      this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);\n\n      this.transcribedText = this.transcribedText.slice(userQuestion.length);\n      handle.markUserCommitted();\n    };\n\n    // wait for the playHandle to finish and check every 1s if user question should be committed\n    commitUserQuestionIfNeeded();\n\n    while (!joinFut.done) {\n      await new Promise<void>(async (resolve) => {\n        setTimeout(resolve, 500);\n        await joinFut.await;\n        resolve();\n      });\n      commitUserQuestionIfNeeded();\n      if (handle.interrupted) break;\n    }\n    commitUserQuestionIfNeeded();\n\n    let collectedText = this.#agentTranscribedText;\n    const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n    const interrupted = handle.interrupted;\n\n    if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {\n      if (handle.extraToolsMessages) {\n        this.chatCtx.messages.push(...handle.extraToolsMessages);\n      }\n      if (interrupted) {\n        collectedText += '…';\n      }\n\n      const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });\n      this.chatCtx.messages.push(msg);\n\n      handle.markSpeechCommitted();\n      if (interrupted) {\n        this.emit(VPAEvent.AGENT_SPEECH_INTERRUPTED, msg);\n      } else {\n        this.emit(VPAEvent.AGENT_SPEECH_COMMITTED, msg);\n      }\n\n      this.#logger\n        .child({\n          agentTranscript: collectedText,\n          interrupted,\n          speechId: handle.id,\n        })\n        .debug('committed agent speech');\n\n      handle.setDone();\n    }\n\n    const executeFunctionCalls = async () => {\n      // if the answer is using tools, execute the functions and automatically generate\n      // a response to the user question from the returned values\n      if (!isUsingTools || interrupted) return;\n\n      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {\n        this.#logger\n          .child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth })\n          .warn('max function calls nested depth reached');\n        return;\n      }\n\n      if (userQuestion && !handle.userCommitted) {\n        throw new Error('user speech should have been committed before using tools');\n      }\n      const llmStream = handle.source;\n      const newFunctionCalls = llmStream.functionCalls;\n\n      new AgentCallContext(this, llmStream);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);\n      const calledFuncs: FunctionCallInfo[] = [];\n      for (const func of newFunctionCalls) {\n        const task = func.func.execute(func.params).then(\n          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),\n          (error) => ({ name: func.name, toolCallId: func.toolCallId, error }),\n        );\n        calledFuncs.push({ ...func, task });\n        this.#logger\n          .child({ function: func.name, speechId: handle.id })\n          .debug('executing AI function');\n        try {\n          await task;\n        } catch {\n          this.#logger\n            .child({ function: func.name, speechId: handle.id })\n            .error('error executing AI function');\n        }\n      }\n\n      const toolCallsInfo = [];\n      const toolCallsResults = [];\n      for (const fnc of calledFuncs) {\n        // ignore the function calls that return void\n        const task = await fnc.task;\n        if (!task || task.result === undefined) continue;\n        toolCallsInfo.push(fnc);\n        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));\n      }\n\n      if (!toolCallsInfo.length) return;\n\n      // generate an answer from the tool calls\n      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];\n      extraToolsMessages.push(...toolCallsResults);\n\n      // create a nested speech handle\n      const newSpeechHandle = SpeechHandle.createToolSpeech(\n        handle.allowInterruptions,\n        handle.addToChatCtx,\n        handle.fncNestedDepth + 1,\n        extraToolsMessages,\n      );\n\n      // synthesize the tool speech with the chat ctx from llmStream\n      const chatCtx = handle.source.chatCtx.copy();\n      chatCtx.messages.push(...extraToolsMessages);\n      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);\n\n      const answerLLMStream = this.llm.chat({\n        chatCtx,\n        fncCtx: this.fncCtx,\n      });\n\n      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);\n      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);\n      handle.addNestedSpeech(newSpeechHandle);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);\n    };\n\n    let finished = false;\n    const task = executeFunctionCalls().then(() => {\n      finished = true;\n    });\n    while (!handle.nestedSpeechFinished) {\n      const changed = handle.nestedSpeechChanged();\n      await Promise.race([changed, task]);\n      while (handle.nestedSpeechHandles.length) {\n        const speech = handle.nestedSpeechHandles[0]!;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        handle.nestedSpeechHandles.shift();\n        this.#playingSpeech = handle;\n      }\n\n      handle.nestedSpeechHandles.forEach(() => handle.nestedSpeechHandles.pop());\n      if (finished) {\n        handle.markNestedSpeechFinished();\n      }\n    }\n    handle.setDone();\n  }\n\n  async #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n  ) {\n    this.#room!.localParticipant!.publishTranscription({\n      participantIdentity: participantIdentity,\n      trackSid: trackSid,\n      segments: [\n        {\n          text: text,\n          final: isFinal,\n          id: id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n    const stream = await this.#room!.localParticipant!.streamText({\n      senderIdentity: participantIdentity,\n      topic: TOPIC_TRANSCRIPTION,\n      attributes: {\n        [ATTRIBUTE_TRANSCRIPTION_TRACK_ID]: trackSid,\n        [ATTRIBUTE_TRANSCRIPTION_FINAL]: isFinal.toString(),\n      },\n    });\n    await stream.write(text);\n    await stream.close();\n  }\n\n  #synthesizeAgentSpeech(\n    speechId: string,\n    source: string | LLMStream | AsyncIterable<string>,\n  ): SynthesisHandle {\n    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);\n    // TODO: where possible we would want to use deltas instead of full text segments, esp for LLM streams over the streamText API\n    synchronizer.on('textUpdated', async (text) => {\n      this.#agentTranscribedText = text.text;\n      await this.#publishTranscription(\n        this.#room!.localParticipant!.identity!,\n        this.#agentPublication?.sid ?? '',\n        text.text,\n        text.final,\n        text.id,\n      );\n    });\n\n    if (!this.#agentOutput) {\n      throw new Error('agent output should be initialized when ready');\n    }\n\n    if (source instanceof LLMStream) {\n      source = llmStreamToStringIterable(speechId, source);\n    }\n\n    const ogSource = source;\n    if (!(typeof source === 'string')) {\n      // TODO(nbsp): itertools.tee\n    }\n\n    const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);\n    if (!ttsSource) {\n      throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');\n    }\n\n    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);\n  }\n\n  async #validateReplyIfPossible() {\n    if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {\n      this.#logger\n        .child({ speechId: this.#playingSpeech.id })\n        .debug('skipping validation, agent is speaking and does not allow interruptions');\n      return;\n    }\n\n    if (!this.#pendingAgentReply) {\n      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {\n        return;\n      }\n      this.#synthesizeAgentReply();\n    }\n\n    if (!this.#pendingAgentReply) {\n      throw new Error('pending agent reply is undefined');\n    }\n\n    // in some bad timimg, we could end up with two pushed agent replies inside the speech queue.\n    // so make sure we directly interrupt every reply when validating a new one\n    if (this.#speechQueueOpen.done) {\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        if (!speech.isReply) continue;\n        if (speech.allowInterruptions) speech.interrupt();\n      }\n    }\n\n    this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');\n\n    if (this.#lastSpeechTime) {\n      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;\n      const transcriptionDelay = Math.max(\n        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,\n        0,\n      );\n      const metrics: PipelineEOUMetrics = {\n        timestamp: Date.now(),\n        sequenceId: this.#pendingAgentReply.id,\n        endOfUtteranceDelay: timeSinceLastSpeech,\n        transcriptionDelay,\n      };\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    }\n\n    this.#addSpeechForPlayout(this.#pendingAgentReply);\n    this.#pendingAgentReply = undefined;\n    this.#transcribedInterimText = '';\n  }\n\n  #interruptIfPossible() {\n    if (\n      !this.#playingSpeech ||\n      !this.#playingSpeech.allowInterruptions ||\n      this.#playingSpeech.interrupted\n    ) {\n      return;\n    }\n\n    if (this.#opts.interruptMinWords !== 0) {\n      // check the final/interim transcribed text for the minimum word count\n      // to interrupt the agent speech\n      const interimWords = this.#opts.transcription.wordTokenizer.tokenize(\n        this.#transcribedInterimText,\n      );\n      if (interimWords.length < this.#opts.interruptMinWords) {\n        return;\n      }\n    }\n    this.#playingSpeech.interrupt();\n  }\n\n  #addSpeechForPlayout(handle: SpeechHandle) {\n    this.#speechQueue.put(handle);\n    this.#speechQueue.put(VoicePipelineAgent.FLUSH_SENTINEL);\n    this.#speechQueueOpen.resolve();\n  }\n\n  /** Close the voice assistant. */\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n\n    this.#room?.removeAllListeners(RoomEvent.ParticipantConnected);\n    // TODO(nbsp): await this.#deferredValidation.close()\n  }\n}\n\nasync function* llmStreamToStringIterable(\n  speechId: string,\n  stream: LLMStream,\n): AsyncIterable<string> {\n  const startTime = Date.now();\n  let firstFrame = true;\n  for await (const chunk of stream) {\n    const content = chunk.choices[0]?.delta.content;\n    if (!content) continue;\n\n    if (firstFrame) {\n      firstFrame = false;\n      log()\n        .child({ speechId, elapsed: Math.round(Date.now() - startTime) })\n        .debug('received first LLM token');\n    }\n    yield content;\n  }\n}\n\n/** This class is used to try to find the best time to validate the agent reply. */\nclass DeferredReplyValidation {\n  // if the STT gives us punctuation, we can try to validate the reply faster.\n  readonly PUNCTUATION = '.!?';\n  readonly PUNCTUATION_REDUCE_FACTOR = 0.75;\n  readonly LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech\n  readonly UNLIKELY_ENDPOINT_DELAY = 6000;\n\n  #validateFunc: () => Promise<void>;\n  #validatingPromise?: Promise<void>;\n  #validatingFuture = new Future();\n  #lastFinalTranscript = '';\n  #lastRecvEndOfSpeechTime = 0;\n  #speaking = false;\n  #endOfSpeechDelay: number;\n  #finalTranscriptDelay: number;\n  #turnDetector?: TurnDetector;\n  #agent: VoicePipelineAgent;\n  #abort?: AbortController;\n\n  constructor(\n    validateFunc: () => Promise<void>,\n    minEndpointingDelay: number,\n    agent: VoicePipelineAgent,\n    turnDetector?: TurnDetector,\n  ) {\n    this.#validateFunc = validateFunc;\n    this.#endOfSpeechDelay = minEndpointingDelay;\n    this.#finalTranscriptDelay = minEndpointingDelay;\n    this.#agent = agent;\n    this.#turnDetector = turnDetector;\n  }\n\n  get validating(): boolean {\n    return !this.#validatingFuture.done;\n  }\n\n  onHumanFinalTranscript(transcript: string) {\n    this.#lastFinalTranscript = transcript.trim();\n    if (this.#speaking) return;\n\n    const hasRecentEndOfSpeech =\n      Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;\n    let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;\n    delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;\n\n    this.#run(delay);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanStartOfSpeech(_: VADEvent) {\n    this.#speaking = true;\n    if (this.validating) {\n      this.#abort?.abort();\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanEndOfSpeech(_: VADEvent) {\n    this.#speaking = false;\n    this.#lastRecvEndOfSpeechTime = Date.now();\n\n    if (this.#lastFinalTranscript) {\n      const delay = this.#endWithPunctuation()\n        ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR\n        : 1_000;\n      this.#run(delay);\n    }\n  }\n\n  // TODO(nbsp): aclose\n\n  #endWithPunctuation(): boolean {\n    return (\n      this.#lastFinalTranscript.length > 0 &&\n      this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]!)\n    );\n  }\n\n  #resetStates() {\n    this.#lastFinalTranscript = '';\n    this.#lastRecvEndOfSpeechTime = 0;\n  }\n\n  #run(delay: number) {\n    const runTask = async (delay: number, chatCtx: ChatContext, signal: AbortSignal) => {\n      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {\n        const startTime = Date.now();\n        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);\n        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;\n        const elapsed = Date.now() - startTime;\n        if (eotProb < unlikelyThreshold) {\n          delay = this.UNLIKELY_ENDPOINT_DELAY;\n        }\n        delay = Math.max(0, delay - elapsed);\n      }\n      const timeout = setTimeout(() => {\n        this.#resetStates();\n        this.#validateFunc();\n      }, delay);\n      signal.addEventListener('abort', () => {\n        clearTimeout(timeout);\n      });\n    };\n\n    this.#abort?.abort();\n    this.#abort = new AbortController();\n    this.#validatingFuture = new Future();\n    const detectCtx = this.#agent.chatCtx.copy();\n    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });\n    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);\n  }\n}\n"],"mappings":"AASA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,kBAAkB;AAC3B,OAAO,kBAAkB;AACzB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAOP,SAAS,aAAa,aAAa,UAAU,UAAU,iBAAiB;AACxE,SAAS,WAAW;AAEpB,SAAmB,iBAAiB,kBAAkB,uBAAuB;AAC7E;AAAA,EACE,qBAAqB;AAAA,EACrB,iBAAiB;AAAA,EACjB;AAAA,OACK;AAEP,SAAS,uBAAuB,8BAA8B;AAE9D,SAAS,UAAU,iBAAiB,wBAAwB;AAC5D,SAAS,oBAAoB,oBAAoB,QAAQ,wBAAwB;AACjF,SAAkC,oBAAoB;AAEtD,SAAS,mBAAmB;AAC5B,SAAS,cAAc,yBAAyB;AAChD,SAAS,YAAY,uBAAuB;AAC5C,SAAS,oBAAoB;AAGtB,MAAM,wBAAwB;AACrC,IAAI;AACJ,IAAI;AAYG,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AAVU,SAAAA;AAAA,GAAA;AAgCL,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA,YAAY,oBAAI,IAAiB;AAAA,EACjC,qBAAoC,CAAC;AAAA,EACrC,OAAO;AAAA,EAEP,YAAY,OAA2B,WAAsB;AAC3D,SAAK,SAAS;AACd,SAAK,aAAa;AAClB,qBAAiB,WAAW;AAAA,EAC9B;AAAA,EAEA,OAAO,aAA+B;AACpC,WAAO,iBAAiB;AAAA,EAC1B;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAc,KAAa,OAAY;AACrC,SAAK,UAAU,IAAI,KAAK,KAAK;AAAA,EAC/B;AAAA,EAEA,YAAY,KAAa,YAAiB,QAAW;AACnD,WAAO,KAAK,UAAU,IAAI,GAAG,KAAK;AAAA,EACpC;AAAA,EAEA,IAAI,YAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,oBAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,oBAAoB,SAAsB;AACxC,SAAK,mBAAmB,KAAK,OAAO;AAAA,EACtC;AACF;AAEA,MAAM,2BAA8C,CAClD,OACA,YACc;AACd,SAAO,MAAM,IAAI,KAAK,EAAE,SAAS,QAAQ,MAAM,OAAO,CAAC;AACzD;AAEA,MAAM,2BAA8C,CAElD,GACA,SACmC;AACnC,SAAO;AACT;AA6BA,MAAM,mCAA8D;AAAA,EAClE,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,uBAAuB;AAAA,EAC9C,eAAe,IAAI,mBAAmB,KAAK;AAAA,EAC3C;AACF;AA2CA,MAAM,oBAAgC;AAAA,EACpC,SAAS,IAAI,YAAY;AAAA,EACzB,oBAAoB;AAAA,EACpB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,eAAe;AACjB;AAGO,MAAM,2BAA4B,aAAsD;AAAA;AAAA,EAEpF,6BAA6B;AAAA,EACtC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAElE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,qBAAqB,IAAI,OAAO;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA,kBAAkB;AAAA,EAClB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,OAAO;AAAA,EAC9B,eAAe,IAAI,mBAA4E;AAAA,EAC/F;AAAA,EACA,WAAW;AAAA,EACX;AAAA,EACA,eAAkD;AAAA,EAClD;AAAA,EACA,UAAU,IAAI;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,wBAAwB;AAAA,EAExB,YAEE,KAEA,KAEA,KAEA,KAEA,OAA4B,mBAC5B;AACA,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAE7C,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,iBAAiB,KAAK,GAAG;AAAA,IACrC;AAEA,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,iBAAiB,KAAK,IAAI,uBAAuB,CAAC;AAAA,IAC9D;AAEA,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AAEZ,SAAK,sBAAsB,IAAI;AAAA,MAC7B,KAAK,yBAAyB,KAAK,IAAI;AAAA,MACvC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK,MAAM;AAAA,IACb;AAAA,EACF;AAAA,EAEA,IAAI,SAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,OAAO,KAAsB;AAC/B,SAAK,MAAM,SAAS;AAAA,EACtB;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAEE,MAQA,cAAiD,MACjD;AACA,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,SAAK,KAAK,GAAG,gBAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,KAAK,GAAG,SAAS,mBAAmB,CAAC,YAAY;AACpD,UAAI,CAAC,WAAY;AACjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,YAAY,WAAW,WAAW,CAAC;AAAA,IACzF,CAAC;AAED,SAAK,KAAK,GAAG,SAAS,mBAAmB,CAAC,YAAY;AACpD,YAAM,aAAa,aAAa,WAAW,aAAa,iDAAgB;AACxE,UAAI,CAAC,WAAY;AAEjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,WAAW,CAAC;AAAA,IAClE,CAAC;AAED,SAAK,KAAK,GAAG,aAAa,mBAAmB,CAAC,YAAY;AACxD,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,GAAG,UAAU,sBAAsB,CAACC,iBAAmC;AAE1E,UAAI,KAAK,cAAc;AACrB;AAAA,MACF;AACA,WAAK,iBAAiB,KAAK,MAAMA,aAAY,QAAS;AAAA,IACxD,CAAC;AAED,SAAK,QAAQ;AACb,SAAK,eAAe;AAEpB,QAAI,aAAa;AACf,UAAI,OAAO,gBAAgB,UAAU;AACnC,aAAK,iBAAiB,WAAW;AAAA,MACnC,OAAO;AACL,aAAK,iBAAiB,YAAY,QAAS;AAAA,MAC7C;AAAA,IACF;AAEA,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA,EAGA,MAAM,IACJ,QACA,qBAAqB,MACrB,eAAe,MACQ;AACvB,UAAM,KAAK,mBAAmB;AAE9B,QAAI;AACJ,QAAI;AACJ,QAAI,cAAc;AAChB,oBAAc,iBAAiB,WAAW;AAC1C,UAAI,kBAAkB,WAAW;AAC/B,aAAK,QAAQ,KAAK,0DAA0D;AAAA,MAC9E,WAAW,OAAO,WAAW,UAAU;AACrC,oBAAY;AAAA,MACd,OAAO;AACL,oBAAY;AACZ,iBAAS,IAAI,mBAA2B;AAAA,MAC1C;AAAA,IACF;AAEA,UAAM,YAAY,aAAa,sBAAsB,oBAAoB,YAAY;AACrF,UAAM,kBAAkB,KAAK,uBAAuB,UAAU,IAAI,MAAM;AACxE,cAAU,WAAW,QAAQ,eAAe;AAE5C,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,sBAAsB;AACpE,WAAK,eAAe,gBAAgB,SAAS;AAAA,IAC/C,OAAO;AACL,WAAK,qBAAqB,SAAS;AAAA,IACrC;AAEA,QAAI,eAAe,WAAW;AAC5B,UAAI;AACJ,UAAI,OAAO,WAAW,UAAU;AAC9B,eAAO;AAAA,MACT,OAAO;AACL,eAAO;AACP,yBAAiB,SAAS,WAAW;AACnC,UAAC,OAAsC,IAAI,KAAK;AAChD,kBAAQ;AAAA,QACV;AACA,QAAC,OAAsC,MAAM;AAAA,MAC/C;AAEA,kBAAY,oBAAoB,YAAY,OAAO,EAAE,MAAM,MAAM,SAAS,UAAU,CAAC,CAAC;AACtF,WAAK,QAAQ,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,4CAA4C;AAAA,IACjF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmB,QAAQ,GAAG;AACzC,UAAM,UAAU,CAACC,WAA4C;AAC3D,aAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAxcpE;AAycQ,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AACD,cAAM,IAAI,QAAQ,CAACC,aAAY,WAAWA,UAASD,MAAK,CAAC;AACzD,aAAI,UAAK,UAAL,mBAAY,aAAa;AAC3B,cAAI,CAAC,WAAW;AACd,oBAAM,UAAK,MAAM,qBAAX,mBAA6B,cAAc,EAAE,CAAC,qBAAqB,GAAG,MAAM;AAAA,UACpF;AAAA,QACF;AACA,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,kBAAkB;AACzB,WAAK,iBAAiB,OAAO;AAAA,IAC/B;AAEA,SAAK,mBAAmB,QAAQ,KAAK;AAAA,EACvC;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,eAAe,KAAK,MAAM,mBAAmB,IAAI,mBAAmB,KAAK;AAC9E,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,SAAK,cAAc,IAAI;AAAA,MACrB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK,MAAM;AAAA,IACb;AACA,SAAK,YAAY,GAAG,gBAAgB,iBAAiB,CAAC,UAAU;AAC9D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,qBAAqB,KAAK;AAAA,IACrD,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,oBAAoB,CAAC,UAAU;AACjE,UAAI,CAAC,KAAK,mBAAmB,MAAM;AACjC;AAAA,MACF;AACA,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,2BAA2B;AAAA,MAC7C;AAEA,UAAI,KAAK;AACT,UAAI,KAAK,MAAM,oBAAoB;AACjC,aAAK,KAAK,IAAI,GAAG,IAAI,MAAM,WAAW;AACtC,aAAK,aAAa,QAAQ,eAAe;AAAA,MAC3C;AAEA,UAAI,MAAM,kBAAkB,KAAK,MAAM,yBAAyB;AAC9D,aAAK,qBAAqB;AAAA,MAC5B;AAEA,UAAI,MAAM,uBAAuB,GAAG;AAClC,aAAK,kBAAkB,KAAK,IAAI,IAAI,MAAM;AAAA,MAC5C;AAAA,IACF,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,eAAe,CAAC,UAAU;AAC5D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,mBAAmB,KAAK;AAAA,IACnD,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,oBAAoB,OAAO,UAAU;AACvE,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,mBAAmB,WAAW;AAAA,MACrC;AACA,WAAK,0BAA0B,MAAM,aAAc,CAAC,EAAE;AAEtD,YAAM,KAAK;AAAA,QACT,KAAK,YAAa,YAAY;AAAA,QAC9B,KAAK,YAAa,gBAAiB;AAAA,QACnC,KAAK;AAAA,QACL;AAAA,QACA,KAAK;AAAA,MACP;AAAA,IACF,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,kBAAkB,OAAO,UAAU;AACrE,YAAM,gBAAgB,MAAM,aAAc,CAAC,EAAE;AAC7C,UAAI,CAAC,cAAe;AAEpB,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,mBAAmB,WAAW;AAAA,MACrC;AAEA,WAAK,2BAA2B,KAAK,IAAI;AACzC,WAAK,oBAAoB,KAAK,kBAAkB,MAAM,MAAM;AAE5D,YAAM,KAAK;AAAA,QACT,KAAK,YAAa,YAAY;AAAA,QAC9B,KAAK,YAAa,gBAAiB;AAAA,QACnC,KAAK;AAAA,QACL;AAAA,QACA,KAAK;AAAA,MACP;AAEA,WAAK,mBAAmB;AAExB,UACE,KAAK,MAAM,wBACV,CAAC,KAAK,kBAAkB,KAAK,eAAe,qBAC7C;AACA,aAAK,sBAAsB;AAAA,MAC7B;AAEA,WAAK,oBAAoB,uBAAuB,aAAa;AAE7D,YAAM,QAAQ,KAAK,MAAM,cAAc,cAAc,SAAS,aAAa;AAC3E,UAAI,MAAM,UAAU,GAAG;AAGrB,aAAK,qBAAqB;AAAA,MAC5B;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAO;AApkBf;AAqkBI,SAAK,aAAa,cAAc;AAChC,UAAM,cAAc,IAAI,YAAY,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAC/E,UAAM,QAAQ,gBAAgB,iBAAiB,mBAAmB,WAAW;AAC7E,SAAK,oBAAoB,QAAM,gBAAK,UAAL,mBAAY,qBAAZ,mBAA8B;AAAA,MAC3D;AAAA,MACA,IAAI,oBAAoB,EAAE,QAAQ,YAAY,kBAAkB,CAAC;AAAA;AAGnE,UAAM,eAAe,IAAI,aAAa,WAAW;AACjD,SAAK,eAAe,IAAI,YAAY,cAAc,KAAK,IAAI;AAE3D,iBAAa,GAAG,kBAAkB,iBAAiB,MAAM;AACvD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,UAAU;AAAA,IAC9B,CAAC;AAED,iBAAa,GAAG,kBAAkB,iBAAiB,CAAC,MAAM;AACxD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,WAAW;AAAA,IAC/B,CAAC;AAED,SAAK,mBAAmB,QAAQ;AAEhC,WAAO,MAAM;AACX,YAAM,KAAK,iBAAiB;AAC5B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,aAAK,iBAAiB;AAAA,MACxB;AACA,WAAK,mBAAmB,IAAI,OAAO;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,wBAAwB;AAxmB1B;AAymBI,eAAK,uBAAL,mBAAyB;AACzB,QAAI,KAAK,eAAe,KAAK,YAAY,UAAU;AACjD,WAAK,aAAa,YAAY,GAAG;AAAA,IACnC;AAEA,SAAK,qBAAqB,aAAa;AAAA,MACrC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK;AAAA,IACP;AACA,UAAM,YAAY,KAAK;AACvB,SAAK,kBAAkB,KAAK,sBAAsB,KAAK,iBAAiB,SAAS;AAAA,EACnF;AAAA,EAEA,sBACE,SACA,QAC0B;AAC1B,WAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,UAAI,SAAS;AACX,cAAM,iBAAiB,OAAO;AAAA,MAChC;AAEA,YAAM,YAAY,KAAK,QAAQ,KAAK;AACpC,YAAM,gBAAgB,KAAK;AAC3B,UAAI,iBAAiB,cAAc,aAAa;AAC9C,aACG,CAAC,cAAc,gBAAgB,cAAc,kBAC9C,CAAC,cAAc,iBACf;AAGA,oBAAU,SAAS;AAAA,YACjB,YAAY,OAAO;AAAA,cACjB,MAAM,cAAc,gBAAgB;AAAA,cACpC,MAAM,SAAS;AAAA,YACjB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,gBAAU,SAAS;AAAA,QACjB,YAAY,OAAO;AAAA,UACjB,MAAM,iCAAQ;AAAA,UACd,MAAM,SAAS;AAAA,QACjB,CAAC;AAAA,MACH;AAEA,mBAAa,EAAE,YAAY,OAAQ,GAAG;AAEtC,UAAI;AACF,YAAI,UAAW,SAAQ;AACvB,YAAI,YAAY,MAAM,KAAK,MAAM,kBAAkB,MAAM,SAAS;AAClE,YAAI,cAAc,OAAO;AACvB,2CAAQ;AACR;AAAA,QACF;AAEA,YAAI,UAAW,SAAQ;AAEvB,YAAI,EAAE,qBAAqB,YAAY;AACrC,sBAAa,MAAM,yBAAyB,MAAM,SAAS;AAAA,QAC7D;AAEA,YAAI,OAAQ,aAAa;AACvB;AAAA,QACF;AAEA,cAAM,kBAAkB,KAAK,uBAAuB,OAAQ,IAAI,SAAS;AACzE,eAAQ,WAAW,WAAW,eAAe;AAAA,MAC/C,UAAE;AACA,yBAAiB;AACjB,qBAAa;AAAA,MACf;AACA,cAAQ;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAsB;AACtC,QAAI;AACF,YAAM,OAAO,sBAAsB;AAAA,IACrC,QAAQ;AACN;AAAA,IACF;AACA,UAAM,KAAK,kBAAmB,oBAAoB;AAClD,UAAM,kBAAkB,OAAO;AAC/B,QAAI,gBAAgB,YAAa;AAEjC,UAAM,eAAe,OAAO;AAC5B,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,UAAU,WAAW,KAAK;AAEhC,UAAM,6BAA6B,MAAM;AACvC,UAAI,CAAC,gBAAgB,gBAAgB,eAAe,OAAO,cAAe;AAC1E,YAAME,gBACJ,OAAO,kBAAkB,aAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AAKtE,UACE,OAAO,sBACP,CAACA,iBACD,WAAW,aAAa,KAAK,8BAC7B,CAAC,QAAQ,MACT;AACA;AAAA,MACF;AAEA,WAAK,QAAQ,MAAM,EAAE,gBAAgB,aAAa,CAAC,EAAE,MAAM,2BAA2B;AACtF,YAAM,UAAU,YAAY,OAAO,EAAE,MAAM,cAAc,MAAM,SAAS,KAAK,CAAC;AAC9E,WAAK,QAAQ,SAAS,KAAK,OAAO;AAClC,WAAK,KAAK,+BAAgC,OAAO;AAEjD,WAAK,kBAAkB,KAAK,gBAAgB,MAAM,aAAa,MAAM;AACrE,aAAO,kBAAkB;AAAA,IAC3B;AAGA,+BAA2B;AAE3B,WAAO,CAAC,QAAQ,MAAM;AACpB,YAAM,IAAI,QAAc,OAAO,YAAY;AACzC,mBAAW,SAAS,GAAG;AACvB,cAAM,QAAQ;AACd,gBAAQ;AAAA,MACV,CAAC;AACD,iCAA2B;AAC3B,UAAI,OAAO,YAAa;AAAA,IAC1B;AACA,+BAA2B;AAE3B,QAAI,gBAAgB,KAAK;AACzB,UAAM,eAAe,OAAO,kBAAkB,aAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AACzF,UAAM,cAAc,OAAO;AAE3B,QAAI,OAAO,iBAAiB,CAAC,gBAAgB,OAAO,gBAAgB;AAClE,UAAI,OAAO,oBAAoB;AAC7B,aAAK,QAAQ,SAAS,KAAK,GAAG,OAAO,kBAAkB;AAAA,MACzD;AACA,UAAI,aAAa;AACf,yBAAiB;AAAA,MACnB;AAEA,YAAM,MAAM,YAAY,OAAO,EAAE,MAAM,eAAe,MAAM,SAAS,UAAU,CAAC;AAChF,WAAK,QAAQ,SAAS,KAAK,GAAG;AAE9B,aAAO,oBAAoB;AAC3B,UAAI,aAAa;AACf,aAAK,KAAK,kCAAmC,GAAG;AAAA,MAClD,OAAO;AACL,aAAK,KAAK,gCAAiC,GAAG;AAAA,MAChD;AAEA,WAAK,QACF,MAAM;AAAA,QACL,iBAAiB;AAAA,QACjB;AAAA,QACA,UAAU,OAAO;AAAA,MACnB,CAAC,EACA,MAAM,wBAAwB;AAEjC,aAAO,QAAQ;AAAA,IACjB;AAEA,UAAM,uBAAuB,YAAY;AAGvC,UAAI,CAAC,gBAAgB,YAAa;AAElC,UAAI,OAAO,kBAAkB,KAAK,MAAM,mBAAmB;AACzD,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,IAAI,gBAAgB,OAAO,eAAe,CAAC,EACpE,KAAK,yCAAyC;AACjD;AAAA,MACF;AAEA,UAAI,gBAAgB,CAAC,OAAO,eAAe;AACzC,cAAM,IAAI,MAAM,2DAA2D;AAAA,MAC7E;AACA,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,UAAU;AAEnC,UAAI,iBAAiB,MAAM,SAAS;AAEpC,WAAK,KAAK,kCAAmC,gBAAgB;AAC7D,YAAM,cAAkC,CAAC;AACzC,iBAAW,QAAQ,kBAAkB;AACnC,cAAMC,QAAO,KAAK,KAAK,QAAQ,KAAK,MAAM,EAAE;AAAA,UAC1C,CAAC,YAAY,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,OAAO;AAAA,UACpE,CAAC,WAAW,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,MAAM;AAAA,QACpE;AACA,oBAAY,KAAK,EAAE,GAAG,MAAM,MAAAA,MAAK,CAAC;AAClC,aAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,uBAAuB;AAChC,YAAI;AACF,gBAAMA;AAAA,QACR,QAAQ;AACN,eAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,6BAA6B;AAAA,QACxC;AAAA,MACF;AAEA,YAAM,gBAAgB,CAAC;AACvB,YAAM,mBAAmB,CAAC;AAC1B,iBAAW,OAAO,aAAa;AAE7B,cAAMA,QAAO,MAAM,IAAI;AACvB,YAAI,CAACA,SAAQA,MAAK,WAAW,OAAW;AACxC,sBAAc,KAAK,GAAG;AACtB,yBAAiB,KAAK,YAAY,6BAA6BA,KAAI,CAAC;AAAA,MACtE;AAEA,UAAI,CAAC,cAAc,OAAQ;AAG3B,YAAM,qBAAqB,CAAC,YAAY,gBAAgB,eAAe,aAAa,CAAC;AACrF,yBAAmB,KAAK,GAAG,gBAAgB;AAG3C,YAAM,kBAAkB,aAAa;AAAA,QACnC,OAAO;AAAA,QACP,OAAO;AAAA,QACP,OAAO,iBAAiB;AAAA,QACxB;AAAA,MACF;AAGA,YAAM,UAAU,OAAO,OAAO,QAAQ,KAAK;AAC3C,cAAQ,SAAS,KAAK,GAAG,kBAAkB;AAC3C,cAAQ,SAAS,KAAK,GAAG,iBAAiB,WAAW,EAAE,iBAAiB;AAExE,YAAM,kBAAkB,KAAK,IAAI,KAAK;AAAA,QACpC;AAAA,QACA,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,kBAAkB,KAAK,uBAAuB,gBAAgB,IAAI,eAAe;AACvF,sBAAgB,WAAW,iBAAiB,eAAe;AAC3D,aAAO,gBAAgB,eAAe;AAEtC,WAAK,KAAK,iCAAkC,WAAW;AAAA,IACzD;AAEA,QAAI,WAAW;AACf,UAAM,OAAO,qBAAqB,EAAE,KAAK,MAAM;AAC7C,iBAAW;AAAA,IACb,CAAC;AACD,WAAO,CAAC,OAAO,sBAAsB;AACnC,YAAM,UAAU,OAAO,oBAAoB;AAC3C,YAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;AAClC,aAAO,OAAO,oBAAoB,QAAQ;AACxC,cAAM,SAAS,OAAO,oBAAoB,CAAC;AAC3C,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,eAAO,oBAAoB,MAAM;AACjC,aAAK,iBAAiB;AAAA,MACxB;AAEA,aAAO,oBAAoB,QAAQ,MAAM,OAAO,oBAAoB,IAAI,CAAC;AACzE,UAAI,UAAU;AACZ,eAAO,yBAAyB;AAAA,MAClC;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,MAAM,sBACJ,qBACA,UACA,MACA,SACA,IACA;AACA,SAAK,MAAO,iBAAkB,qBAAqB;AAAA,MACjD;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AACD,UAAM,SAAS,MAAM,KAAK,MAAO,iBAAkB,WAAW;AAAA,MAC5D,gBAAgB;AAAA,MAChB,OAAO;AAAA,MACP,YAAY;AAAA,QACV,CAAC,gCAAgC,GAAG;AAAA,QACpC,CAAC,6BAA6B,GAAG,QAAQ,SAAS;AAAA,MACpD;AAAA,IACF,CAAC;AACD,UAAM,OAAO,MAAM,IAAI;AACvB,UAAM,OAAO,MAAM;AAAA,EACrB;AAAA,EAEA,uBACE,UACA,QACiB;AACjB,UAAM,eAAe,IAAI,sBAAsB,sBAAsB;AAErE,iBAAa,GAAG,eAAe,OAAO,SAAS;AAl6BnD;AAm6BM,WAAK,wBAAwB,KAAK;AAClC,YAAM,KAAK;AAAA,QACT,KAAK,MAAO,iBAAkB;AAAA,UAC9B,UAAK,sBAAL,mBAAwB,QAAO;AAAA,QAC/B,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF,CAAC;AAED,QAAI,CAAC,KAAK,cAAc;AACtB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAEA,QAAI,kBAAkB,WAAW;AAC/B,eAAS,0BAA0B,UAAU,MAAM;AAAA,IACrD;AAEA,UAAM,WAAW;AACjB,QAAI,EAAE,OAAO,WAAW,WAAW;AAAA,IAEnC;AAEA,UAAM,YAAY,KAAK,MAAM,kBAAkB,MAAM,QAAQ;AAC7D,QAAI,CAAC,WAAW;AACd,YAAM,IAAI,MAAM,+DAA+D;AAAA,IACjF;AAEA,WAAO,KAAK,aAAa,WAAW,UAAU,WAAW,YAAY;AAAA,EACvE;AAAA,EAEA,MAAM,2BAA2B;AAC/B,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,oBAAoB;AAClE,WAAK,QACF,MAAM,EAAE,UAAU,KAAK,eAAe,GAAG,CAAC,EAC1C,MAAM,yEAAyE;AAClF;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,UAAI,KAAK,MAAM,uBAAuB,CAAC,KAAK,iBAAiB;AAC3D;AAAA,MACF;AACA,WAAK,sBAAsB;AAAA,IAC7B;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAIA,QAAI,KAAK,iBAAiB,MAAM;AAC9B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,YAAI,CAAC,OAAO,QAAS;AACrB,YAAI,OAAO,mBAAoB,QAAO,UAAU;AAAA,MAClD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,UAAU,KAAK,mBAAmB,GAAG,CAAC,EAAE,MAAM,uBAAuB;AAE1F,QAAI,KAAK,iBAAiB;AACxB,YAAM,sBAAsB,KAAK,IAAI,IAAI,KAAK;AAC9C,YAAM,qBAAqB,KAAK;AAAA,SAC7B,KAAK,4BAA4B,KAAK,KAAK;AAAA,QAC5C;AAAA,MACF;AACA,YAAM,UAA8B;AAAA,QAClC,WAAW,KAAK,IAAI;AAAA,QACpB,YAAY,KAAK,mBAAmB;AAAA,QACpC,qBAAqB;AAAA,QACrB;AAAA,MACF;AACA,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C;AAEA,SAAK,qBAAqB,KAAK,kBAAkB;AACjD,SAAK,qBAAqB;AAC1B,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,uBAAuB;AACrB,QACE,CAAC,KAAK,kBACN,CAAC,KAAK,eAAe,sBACrB,KAAK,eAAe,aACpB;AACA;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,sBAAsB,GAAG;AAGtC,YAAM,eAAe,KAAK,MAAM,cAAc,cAAc;AAAA,QAC1D,KAAK;AAAA,MACP;AACA,UAAI,aAAa,SAAS,KAAK,MAAM,mBAAmB;AACtD;AAAA,MACF;AAAA,IACF;AACA,SAAK,eAAe,UAAU;AAAA,EAChC;AAAA,EAEA,qBAAqB,QAAsB;AACzC,SAAK,aAAa,IAAI,MAAM;AAC5B,SAAK,aAAa,IAAI,mBAAmB,cAAc;AACvD,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA;AAAA,EAGA,MAAM,QAAQ;AAlhChB;AAmhCI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AAEA,eAAK,UAAL,mBAAY,mBAAmB,UAAU;AAAA,EAE3C;AACF;AAEA,gBAAgB,0BACd,UACA,QACuB;AA/hCzB;AAgiCE,QAAM,YAAY,KAAK,IAAI;AAC3B,MAAI,aAAa;AACjB,mBAAiB,SAAS,QAAQ;AAChC,UAAM,WAAU,WAAM,QAAQ,CAAC,MAAf,mBAAkB,MAAM;AACxC,QAAI,CAAC,QAAS;AAEd,QAAI,YAAY;AACd,mBAAa;AACb,UAAI,EACD,MAAM,EAAE,UAAU,SAAS,KAAK,MAAM,KAAK,IAAI,IAAI,SAAS,EAAE,CAAC,EAC/D,MAAM,0BAA0B;AAAA,IACrC;AACA,UAAM;AAAA,EACR;AACF;AAGA,MAAM,wBAAwB;AAAA;AAAA,EAEnB,cAAc;AAAA,EACd,4BAA4B;AAAA,EAC5B,4BAA4B;AAAA;AAAA,EAC5B,0BAA0B;AAAA,EAEnC;AAAA,EACA;AAAA,EACA,oBAAoB,IAAI,OAAO;AAAA,EAC/B,uBAAuB;AAAA,EACvB,2BAA2B;AAAA,EAC3B,YAAY;AAAA,EACZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,cACA,qBACA,OACA,cACA;AACA,SAAK,gBAAgB;AACrB,SAAK,oBAAoB;AACzB,SAAK,wBAAwB;AAC7B,SAAK,SAAS;AACd,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,CAAC,KAAK,kBAAkB;AAAA,EACjC;AAAA,EAEA,uBAAuB,YAAoB;AACzC,SAAK,uBAAuB,WAAW,KAAK;AAC5C,QAAI,KAAK,UAAW;AAEpB,UAAM,uBACJ,KAAK,IAAI,IAAI,KAAK,2BAA2B,KAAK;AACpD,QAAI,QAAQ,uBAAuB,KAAK,oBAAoB,KAAK;AACjE,YAAQ,KAAK,oBAAoB,IAAI,QAAQ,KAAK,4BAA4B;AAE9E,SAAK,KAAK,KAAK;AAAA,EACjB;AAAA;AAAA,EAGA,qBAAqB,GAAa;AAlmCpC;AAmmCI,SAAK,YAAY;AACjB,QAAI,KAAK,YAAY;AACnB,iBAAK,WAAL,mBAAa;AAAA,IACf;AAAA,EACF;AAAA;AAAA,EAGA,mBAAmB,GAAa;AAC9B,SAAK,YAAY;AACjB,SAAK,2BAA2B,KAAK,IAAI;AAEzC,QAAI,KAAK,sBAAsB;AAC7B,YAAM,QAAQ,KAAK,oBAAoB,IACnC,KAAK,oBAAoB,KAAK,4BAC9B;AACJ,WAAK,KAAK,KAAK;AAAA,IACjB;AAAA,EACF;AAAA;AAAA,EAIA,sBAA+B;AAC7B,WACE,KAAK,qBAAqB,SAAS,KACnC,KAAK,YAAY,SAAS,KAAK,qBAAqB,KAAK,qBAAqB,SAAS,CAAC,CAAE;AAAA,EAE9F;AAAA,EAEA,eAAe;AACb,SAAK,uBAAuB;AAC5B,SAAK,2BAA2B;AAAA,EAClC;AAAA,EAEA,KAAK,OAAe;AApoCtB;AAqoCI,UAAM,UAAU,OAAOH,QAAe,SAAsB,WAAwB;AAClF,UAAI,KAAK,wBAAwB,CAAC,KAAK,aAAa,KAAK,eAAe;AACtE,cAAM,YAAY,KAAK,IAAI;AAC3B,cAAM,UAAU,MAAM,KAAK,cAAc,iBAAiB,OAAO;AACjE,cAAM,oBAAoB,KAAK,cAAc;AAC7C,cAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAI,UAAU,mBAAmB;AAC/B,UAAAA,SAAQ,KAAK;AAAA,QACf;AACA,QAAAA,SAAQ,KAAK,IAAI,GAAGA,SAAQ,OAAO;AAAA,MACrC;AACA,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,aAAa;AAClB,aAAK,cAAc;AAAA,MACrB,GAAGA,MAAK;AACR,aAAO,iBAAiB,SAAS,MAAM;AACrC,qBAAa,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAEA,eAAK,WAAL,mBAAa;AACb,SAAK,SAAS,IAAI,gBAAgB;AAClC,SAAK,oBAAoB,IAAI,OAAO;AACpC,UAAM,YAAY,KAAK,OAAO,QAAQ,KAAK;AAC3C,cAAU,OAAO,EAAE,MAAM,KAAK,OAAO,iBAAiB,MAAM,SAAS,KAAK,CAAC;AAC3E,SAAK,qBAAqB,QAAQ,OAAO,WAAW,KAAK,OAAO,MAAM;AAAA,EACxE;AACF;","names":["VPAEvent","participant","delay","resolve","isUsingTools","task"]}
+{"version":3,"sources":["../../src/pipeline/pipeline_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  LocalTrackPublication,\n  NoiseCancellationOptions,\n  RemoteParticipant,\n  Room,\n} from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { randomUUID } from 'node:crypto';\nimport EventEmitter from 'node:events';\nimport {\n  ATTRIBUTE_SEGMENT_ID,\n  ATTRIBUTE_TRANSCRIPTION_FINAL,\n  ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n  TOPIC_TRANSCRIPTION,\n} from '../constants.js';\nimport type {\n  CallableFunctionResult,\n  FunctionCallInfo,\n  FunctionContext,\n  LLM,\n} from '../llm/index.js';\nimport { ChatContext, ChatMessage, ChatRole, LLMEvent, LLMStream } from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { AgentMetrics, PipelineEOUMetrics } from '../metrics/base.js';\nimport { type STT, StreamAdapter as STTStreamAdapter, SpeechEventType } from '../stt/index.js';\nimport {\n  SentenceTokenizer as BasicSentenceTokenizer,\n  WordTokenizer as BasicWordTokenizer,\n  hyphenateWord,\n} from '../tokenize/basic/index.js';\nimport type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';\nimport { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';\nimport type { TTS } from '../tts/index.js';\nimport { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { SpeechSource, SynthesisHandle } from './agent_output.js';\nimport { AgentOutput } from './agent_output.js';\nimport { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';\nimport { HumanInput, HumanInputEvent } from './human_input.js';\nimport { SpeechHandle } from './speech_handle.js';\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\nlet lastSpeechData: { sequenceId: string } | undefined;\nlet speechData: { sequenceId: string } | undefined;\n\nexport type BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n) => LLMStream | false | void | Promise<LLMStream | false | void>;\n\nexport type BeforeTTSCallback = (\n  agent: VoicePipelineAgent,\n  source: string | AsyncIterable<string>,\n) => SpeechSource;\n\nexport enum VPAEvent {\n  USER_STARTED_SPEAKING,\n  USER_STOPPED_SPEAKING,\n  AGENT_STARTED_SPEAKING,\n  AGENT_STOPPED_SPEAKING,\n  USER_SPEECH_COMMITTED,\n  AGENT_SPEECH_COMMITTED,\n  AGENT_SPEECH_INTERRUPTED,\n  FUNCTION_CALLS_COLLECTED,\n  FUNCTION_CALLS_FINISHED,\n  METRICS_COLLECTED,\n}\n\nexport type VPACallbacks = {\n  [VPAEvent.USER_STARTED_SPEAKING]: () => void;\n  [VPAEvent.USER_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STARTED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.USER_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;\n  [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;\n  [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;\n  [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;\n};\n\ninterface TurnDetector {\n  unlikelyThreshold: number;\n  supportsLanguage: (language?: string) => boolean;\n  predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;\n}\n\nexport class AgentCallContext {\n  #agent: VoicePipelineAgent;\n  #llmStream: LLMStream;\n  #metadata = new Map<string, any>();\n  #extraChatMessages: ChatMessage[] = [];\n  static #current: AgentCallContext;\n\n  constructor(agent: VoicePipelineAgent, llmStream: LLMStream) {\n    this.#agent = agent;\n    this.#llmStream = llmStream;\n    AgentCallContext.#current = this;\n  }\n\n  static getCurrent(): AgentCallContext {\n    return AgentCallContext.#current;\n  }\n\n  get agent(): VoicePipelineAgent {\n    return this.#agent;\n  }\n\n  storeMetadata(key: string, value: any) {\n    this.#metadata.set(key, value);\n  }\n\n  getMetadata(key: string, orDefault: any = undefined) {\n    return this.#metadata.get(key) || orDefault;\n  }\n\n  get llmStream(): LLMStream {\n    return this.#llmStream;\n  }\n\n  get extraChatMessages() {\n    return this.#extraChatMessages;\n  }\n\n  addExtraChatMessage(message: ChatMessage) {\n    this.#extraChatMessages.push(message);\n  }\n}\n\nconst defaultBeforeLLMCallback: BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n): LLMStream => {\n  return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });\n};\n\nconst defaultBeforeTTSCallback: BeforeTTSCallback = (\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  _: VoicePipelineAgent,\n  text: string | AsyncIterable<string>,\n): string | AsyncIterable<string> => {\n  return text;\n};\n\nexport interface AgentTranscriptionOptions {\n  /** Whether to forward the user transcription to the client */\n  userTranscription: boolean;\n  /** Whether to forward the agent transcription to the client */\n  agentTranscription: boolean;\n  /**\n   * The speed at which the agent's speech transcription is forwarded to the client.\n   * We try to mimic the agent's speech speed by adjusting the transcription speed.\n   */\n  agentTranscriptionSpeech: number;\n  /**\n   * The tokenizer used to split the speech into sentences.\n   * This is used to decide when to mark a transcript as final for the agent transcription.\n   */\n  sentenceTokenizer: SentenceTokenizer;\n  /**\n   * The tokenizer used to split the speech into words.\n   * This is used to simulate the \"interim results\" of the agent transcription.\n   */\n  wordTokenizer: WordTokenizer;\n  /**\n   * A function that takes a string (word) as input and returns a list of strings,\n   * representing the hyphenated parts of the word.\n   */\n  hyphenateWord: (word: string) => string[];\n}\n\nconst defaultAgentTranscriptionOptions: AgentTranscriptionOptions = {\n  userTranscription: true,\n  agentTranscription: true,\n  agentTranscriptionSpeech: 1,\n  sentenceTokenizer: new BasicSentenceTokenizer(),\n  wordTokenizer: new BasicWordTokenizer(false),\n  hyphenateWord: hyphenateWord,\n};\n\nexport interface VPAOptions {\n  /** Chat context for the assistant. */\n  chatCtx?: ChatContext;\n  /** Function context for the assistant. */\n  fncCtx?: FunctionContext;\n  /** Whether to allow the user to interrupt the assistant. */\n  allowInterruptions: boolean;\n  /** Minimum duration of speech to consider for interruption. */\n  interruptSpeechDuration: number;\n  /** Minimum number of words to consider for interuption. This may increase latency. */\n  interruptMinWords: number;\n  /** Delay to wait before considering the user speech done. */\n  minEndpointingDelay: number;\n  maxNestedFncCalls: number;\n  /* Whether to preemptively synthesize responses. */\n  preemptiveSynthesis: boolean;\n  /*\n   * Callback called when the assistant is about to synthesize a reply.\n   *\n   * @remarks\n   * Returning void will create a default LLM stream.\n   * You can also return your own LLM stream by calling `llm.chat()`.\n   * Returning `false` ill cancel the synthesis of the reply.\n   */\n  beforeLLMCallback: BeforeLLMCallback;\n  /*\n   * Callback called when the assistant is about to synthesize speech.\n   *\n   * @remarks\n   * This can be used to customize text before synthesis\n   * (e.g. editing the pronunciation of a word).\n   */\n  beforeTTSCallback: BeforeTTSCallback;\n  /** Options for assistant transcription. */\n  transcription: AgentTranscriptionOptions;\n  /** Turn detection model to use. */\n  turnDetector?: TurnDetector;\n  /** Noise cancellation options. */\n  noiseCancellation?: NoiseCancellationOptions;\n}\n\nconst defaultVPAOptions: VPAOptions = {\n  chatCtx: new ChatContext(),\n  allowInterruptions: true,\n  interruptSpeechDuration: 50,\n  interruptMinWords: 0,\n  minEndpointingDelay: 500,\n  maxNestedFncCalls: 1,\n  preemptiveSynthesis: false,\n  beforeLLMCallback: defaultBeforeLLMCallback,\n  beforeTTSCallback: defaultBeforeTTSCallback,\n  transcription: defaultAgentTranscriptionOptions,\n};\n\n/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */\nexport class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<VPACallbacks>) {\n  /** Minimum time played for the user speech to be committed to the chat context. */\n  readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n\n  #vad: VAD;\n  #stt: STT;\n  #llm: LLM;\n  #tts: TTS;\n  #opts: VPAOptions;\n  #humanInput?: HumanInput;\n  #agentOutput?: AgentOutput;\n  #trackPublishedFut = new Future();\n  #pendingAgentReply?: SpeechHandle;\n  #agentReplyTask?: CancellablePromise<void>;\n  #playingSpeech?: SpeechHandle;\n  transcribedText = '';\n  #transcribedInterimText = '';\n  #speechQueueOpen = new Future();\n  #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();\n  #updateStateTask?: CancellablePromise<void>;\n  #started = false;\n  #room?: Room;\n  #participant: RemoteParticipant | string | null = null;\n  #deferredValidation: DeferredReplyValidation;\n  #logger = log();\n  #agentPublication?: LocalTrackPublication;\n  #lastFinalTranscriptTime?: number;\n  #lastSpeechTime?: number;\n  #transcriptionId?: string;\n  #agentTranscribedText = '';\n\n  constructor(\n    /** Voice Activity Detection instance. */\n    vad: VAD,\n    /** Speech-to-Text instance. */\n    stt: STT,\n    /** Large Language Model instance. */\n    llm: LLM,\n    /** Text-to-Speech instance. */\n    tts: TTS,\n    /** Additional VoicePipelineAgent options. */\n    opts: Partial<VPAOptions> = defaultVPAOptions,\n  ) {\n    super();\n\n    this.#opts = { ...defaultVPAOptions, ...opts };\n\n    if (!stt.capabilities.streaming) {\n      stt = new STTStreamAdapter(stt, vad);\n    }\n\n    if (!tts.capabilities.streaming) {\n      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());\n    }\n\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#llm = llm;\n    this.#tts = tts;\n\n    this.#deferredValidation = new DeferredReplyValidation(\n      this.#validateReplyIfPossible.bind(this),\n      this.#opts.minEndpointingDelay,\n      this,\n      this.#opts.turnDetector,\n    );\n  }\n\n  get fncCtx(): FunctionContext | undefined {\n    return this.#opts.fncCtx;\n  }\n\n  set fncCtx(ctx: FunctionContext) {\n    this.#opts.fncCtx = ctx;\n  }\n\n  get chatCtx(): ChatContext {\n    return this.#opts.chatCtx!;\n  }\n\n  get llm(): LLM {\n    return this.#llm;\n  }\n\n  get tts(): TTS {\n    return this.#tts;\n  }\n\n  get stt(): STT {\n    return this.#stt;\n  }\n\n  get vad(): VAD {\n    return this.#vad;\n  }\n\n  /** Start the voice assistant. */\n  start(\n    /** The room to connect to. */\n    room: Room,\n    /**\n     * The participant to listen to.\n     *\n     * @remarks\n     * Can be a participant or an identity.\n     * If omitted, the first participant in the room will be selected.\n     */\n    participant: RemoteParticipant | string | null = null,\n  ) {\n    if (this.#started) {\n      throw new Error('voice assistant already started');\n    }\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {\n      if (!speechData) return;\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });\n    });\n\n    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {\n      const sequenceId = speechData ? speechData.sequenceId : lastSpeechData?.sequenceId;\n      if (!sequenceId) return;\n\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId });\n    });\n\n    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n      // automatically link to the first participant that connects, if not already linked\n      if (this.#participant) {\n        return;\n      }\n      this.#linkParticipant.call(this, participant.identity!);\n    });\n\n    this.#room = room;\n    this.#participant = participant;\n\n    if (participant) {\n      if (typeof participant === 'string') {\n        this.#linkParticipant(participant);\n      } else {\n        this.#linkParticipant(participant.identity!);\n      }\n    }\n\n    this.#run();\n  }\n\n  /** Play a speech source through the voice assistant. */\n  async say(\n    source: string | LLMStream | AsyncIterable<string>,\n    allowInterruptions = true,\n    addToChatCtx = true,\n  ): Promise<SpeechHandle> {\n    await this.#trackPublishedFut.await;\n\n    let callContext: AgentCallContext | undefined;\n    let fncSource: string | AsyncIterable<string> | undefined;\n    if (addToChatCtx) {\n      callContext = AgentCallContext.getCurrent();\n      if (source instanceof LLMStream) {\n        this.#logger.warn('LLMStream will be ignored for function call chat context');\n      } else if (typeof source === 'string') {\n        fncSource = source;\n      } else {\n        fncSource = source;\n        source = new AsyncIterableQueue<string>();\n      }\n    }\n\n    const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);\n    const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);\n    newHandle.initialize(source, synthesisHandle);\n\n    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {\n      this.#playingSpeech.addNestedSpeech(newHandle);\n    } else {\n      this.#addSpeechForPlayout(newHandle);\n    }\n\n    if (callContext && fncSource) {\n      let text: string;\n      if (typeof source === 'string') {\n        text = fncSource as string;\n      } else {\n        text = '';\n        for await (const chunk of fncSource) {\n          (source as AsyncIterableQueue<string>).put(chunk);\n          text += chunk;\n        }\n        (source as AsyncIterableQueue<string>).close();\n      }\n\n      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));\n      this.#logger.child({ text }).debug('added speech to function call chat context');\n    }\n\n    return newHandle;\n  }\n\n  #updateState(state: AgentState, delay = 0) {\n    const runTask = (delay: number): CancellablePromise<void> => {\n      return new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n        await new Promise((resolve) => setTimeout(resolve, delay));\n        if (this.#room?.isConnected) {\n          if (!cancelled) {\n            await this.#room.localParticipant?.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state });\n          }\n        }\n        resolve();\n      });\n    };\n\n    if (this.#updateStateTask) {\n      this.#updateStateTask.cancel();\n    }\n\n    this.#updateStateTask = runTask(delay);\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.#room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.#participant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    this.#humanInput = new HumanInput(\n      this.#room,\n      this.#vad,\n      this.#stt,\n      this.#participant,\n      this.#opts.noiseCancellation,\n    );\n    this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STARTED_SPEAKING);\n      this.#deferredValidation.onHumanStartOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {\n      if (!this.#trackPublishedFut.done) {\n        return;\n      }\n      if (!this.#agentOutput) {\n        throw new Error('agent output is undefined');\n      }\n\n      let tv = 1;\n      if (this.#opts.allowInterruptions) {\n        tv = Math.max(0, 1 - event.probability);\n        this.#agentOutput.playout.targetVolume = tv;\n      }\n\n      if (event.speechDuration >= this.#opts.interruptSpeechDuration) {\n        this.#interruptIfPossible();\n      }\n\n      if (event.rawAccumulatedSpeech > 0) {\n        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;\n      }\n    });\n    this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STOPPED_SPEAKING);\n      this.#deferredValidation.onHumanEndOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, async (event) => {\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n      this.#transcribedInterimText = event.alternatives![0].text;\n\n      await this.#publishTranscription(\n        this.#humanInput!.participant.identity,\n        this.#humanInput!.subscribedTrack!.sid!,\n        this.#transcribedInterimText,\n        false,\n        this.#transcriptionId,\n        this.#transcriptionId,\n      );\n    });\n    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, async (event) => {\n      const newTranscript = event.alternatives![0].text;\n      if (!newTranscript) return;\n\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n\n      this.#lastFinalTranscriptTime = Date.now();\n      this.transcribedText += (this.transcribedText ? ' ' : '') + newTranscript;\n\n      await this.#publishTranscription(\n        this.#humanInput!.participant.identity,\n        this.#humanInput!.subscribedTrack!.sid!,\n        this.transcribedText,\n        true,\n        this.#transcriptionId,\n        this.#transcriptionId,\n      );\n\n      this.#transcriptionId = undefined;\n\n      if (\n        this.#opts.preemptiveSynthesis &&\n        (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)\n      ) {\n        this.#synthesizeAgentReply();\n      }\n\n      this.#deferredValidation.onHumanFinalTranscript(newTranscript);\n\n      const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);\n      if (words.length >= 3) {\n        // VAD can sometimes not detect that the human is speaking.\n        // to make the interruption more reliable, we also interrupt on the final transcript.\n        this.#interruptIfPossible();\n      }\n    });\n  }\n\n  async #run() {\n    this.#updateState('initializing');\n    const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);\n    const track = LocalAudioTrack.createAudioTrack('assistant_voice', audioSource);\n    this.#agentPublication = await this.#room?.localParticipant?.publishTrack(\n      track,\n      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n    );\n\n    const agentPlayout = new AgentPlayout(audioSource);\n    this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);\n\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {\n      this.emit(VPAEvent.AGENT_STARTED_SPEAKING);\n      this.#updateState('speaking');\n    });\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {\n      this.emit(VPAEvent.AGENT_STOPPED_SPEAKING);\n      this.#updateState('listening');\n    });\n\n    this.#trackPublishedFut.resolve();\n\n    while (true) {\n      await this.#speechQueueOpen.await;\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        this.#playingSpeech = undefined;\n      }\n      this.#speechQueueOpen = new Future();\n    }\n  }\n\n  #synthesizeAgentReply() {\n    this.#pendingAgentReply?.cancel();\n    if (this.#humanInput && this.#humanInput.speaking) {\n      this.#updateState('thinking', 200);\n    }\n\n    this.#pendingAgentReply = SpeechHandle.createAssistantReply(\n      this.#opts.allowInterruptions,\n      true,\n      this.transcribedText,\n    );\n    const newHandle = this.#pendingAgentReply;\n    this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);\n  }\n\n  #synthesizeAnswerTask(\n    oldTask: CancellablePromise<void> | undefined,\n    handle?: SpeechHandle,\n  ): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      const copiedCtx = this.chatCtx.copy();\n      const playingSpeech = this.#playingSpeech;\n      if (playingSpeech && playingSpeech.initialized) {\n        if (\n          (!playingSpeech.userQuestion || playingSpeech.userCommitted) &&\n          !playingSpeech.speechCommitted\n        ) {\n          // the speech is playing but not committed yet,\n          // add it to the chat context for this new reply synthesis\n          copiedCtx.messages.push(\n            ChatMessage.create({\n              text: playingSpeech.synthesisHandle.text,\n              role: ChatRole.ASSISTANT,\n            }),\n          );\n        }\n      }\n\n      copiedCtx.messages.push(\n        ChatMessage.create({\n          text: handle?.userQuestion,\n          role: ChatRole.USER,\n        }),\n      );\n\n      speechData = { sequenceId: handle!.id };\n\n      try {\n        if (cancelled) resolve();\n        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);\n        if (llmStream === false) {\n          handle?.cancel();\n          return;\n        }\n\n        if (cancelled) resolve();\n        // fallback to default impl if no custom/user stream is returned\n        if (!(llmStream instanceof LLMStream)) {\n          llmStream = (await defaultBeforeLLMCallback(this, copiedCtx)) as LLMStream;\n        }\n\n        if (handle!.interrupted) {\n          return;\n        }\n\n        const synthesisHandle = this.#synthesizeAgentSpeech(handle!.id, llmStream);\n        handle!.initialize(llmStream, synthesisHandle);\n      } finally {\n        lastSpeechData = speechData;\n        speechData = undefined;\n      }\n      resolve();\n    });\n  }\n\n  async #playSpeech(handle: SpeechHandle) {\n    try {\n      await handle.waitForInitialization();\n    } catch {\n      return;\n    }\n    await this.#agentPublication!.waitForSubscription();\n    const synthesisHandle = handle.synthesisHandle;\n    if (synthesisHandle.interrupted) return;\n\n    const userQuestion = handle.userQuestion;\n    const playHandle = synthesisHandle.play();\n    const joinFut = playHandle.join();\n\n    const commitUserQuestionIfNeeded = () => {\n      if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted) return;\n      const isUsingTools =\n        handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n\n      // make sure at least some speech was played before committing the user message\n      // since we try to validate as fast as possible it is possible the agent gets interrupted\n      // really quickly (barely audible), we don't want to mark this question as \"answered\".\n      if (\n        handle.allowInterruptions &&\n        !isUsingTools &&\n        playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT &&\n        !joinFut.done\n      ) {\n        return;\n      }\n\n      this.#logger.child({ userTranscript: userQuestion }).debug('committed user transcript');\n      const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });\n      this.chatCtx.messages.push(userMsg);\n      this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);\n\n      this.transcribedText = this.transcribedText.slice(userQuestion.length);\n      handle.markUserCommitted();\n    };\n\n    // wait for the playHandle to finish and check every 1s if user question should be committed\n    commitUserQuestionIfNeeded();\n\n    while (!joinFut.done) {\n      await new Promise<void>(async (resolve) => {\n        setTimeout(resolve, 500);\n        await joinFut.await;\n        resolve();\n      });\n      commitUserQuestionIfNeeded();\n      if (handle.interrupted) break;\n    }\n    commitUserQuestionIfNeeded();\n\n    let collectedText = this.#agentTranscribedText;\n    const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n    const interrupted = handle.interrupted;\n\n    if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {\n      if (handle.extraToolsMessages) {\n        this.chatCtx.messages.push(...handle.extraToolsMessages);\n      }\n      if (interrupted) {\n        collectedText += '…';\n      }\n\n      const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });\n      this.chatCtx.messages.push(msg);\n\n      handle.markSpeechCommitted();\n      if (interrupted) {\n        this.emit(VPAEvent.AGENT_SPEECH_INTERRUPTED, msg);\n      } else {\n        this.emit(VPAEvent.AGENT_SPEECH_COMMITTED, msg);\n      }\n\n      this.#logger\n        .child({\n          agentTranscript: collectedText,\n          interrupted,\n          speechId: handle.id,\n        })\n        .debug('committed agent speech');\n\n      handle.setDone();\n    }\n\n    const executeFunctionCalls = async () => {\n      // if the answer is using tools, execute the functions and automatically generate\n      // a response to the user question from the returned values\n      if (!isUsingTools || interrupted) return;\n\n      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {\n        this.#logger\n          .child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth })\n          .warn('max function calls nested depth reached');\n        return;\n      }\n\n      if (userQuestion && !handle.userCommitted) {\n        throw new Error('user speech should have been committed before using tools');\n      }\n      const llmStream = handle.source;\n      const newFunctionCalls = llmStream.functionCalls;\n\n      new AgentCallContext(this, llmStream);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);\n      const calledFuncs: FunctionCallInfo[] = [];\n      for (const func of newFunctionCalls) {\n        const task = func.func.execute(func.params).then(\n          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),\n          (error) => ({ name: func.name, toolCallId: func.toolCallId, error }),\n        );\n        calledFuncs.push({ ...func, task });\n        this.#logger\n          .child({ function: func.name, speechId: handle.id })\n          .debug('executing AI function');\n        try {\n          await task;\n        } catch {\n          this.#logger\n            .child({ function: func.name, speechId: handle.id })\n            .error('error executing AI function');\n        }\n      }\n\n      const toolCallsInfo = [];\n      const toolCallsResults = [];\n      for (const fnc of calledFuncs) {\n        // ignore the function calls that return void\n        const task = await fnc.task;\n        if (!task || task.result === undefined) continue;\n        toolCallsInfo.push(fnc);\n        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));\n      }\n\n      if (!toolCallsInfo.length) return;\n\n      // generate an answer from the tool calls\n      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];\n      extraToolsMessages.push(...toolCallsResults);\n\n      // create a nested speech handle\n      const newSpeechHandle = SpeechHandle.createToolSpeech(\n        handle.allowInterruptions,\n        handle.addToChatCtx,\n        handle.fncNestedDepth + 1,\n        extraToolsMessages,\n      );\n\n      // synthesize the tool speech with the chat ctx from llmStream\n      const chatCtx = handle.source.chatCtx.copy();\n      chatCtx.messages.push(...extraToolsMessages);\n      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);\n\n      const answerLLMStream = this.llm.chat({\n        chatCtx,\n        fncCtx: this.fncCtx,\n      });\n\n      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);\n      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);\n      handle.addNestedSpeech(newSpeechHandle);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);\n    };\n\n    let finished = false;\n    const task = executeFunctionCalls().then(() => {\n      finished = true;\n    });\n    while (!handle.nestedSpeechFinished) {\n      const changed = handle.nestedSpeechChanged();\n      await Promise.race([changed, task]);\n      while (handle.nestedSpeechHandles.length) {\n        const speech = handle.nestedSpeechHandles[0]!;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        handle.nestedSpeechHandles.shift();\n        this.#playingSpeech = handle;\n      }\n\n      handle.nestedSpeechHandles.forEach(() => handle.nestedSpeechHandles.pop());\n      if (finished) {\n        handle.markNestedSpeechFinished();\n      }\n    }\n    handle.setDone();\n  }\n\n  async #publishTranscription(\n    participantIdentity: string,\n    trackSid: string,\n    text: string,\n    isFinal: boolean,\n    id: string,\n    segmentId: string,\n  ) {\n    this.#room!.localParticipant!.publishTranscription({\n      participantIdentity: participantIdentity,\n      trackSid: trackSid,\n      segments: [\n        {\n          text: text,\n          final: isFinal,\n          id: id,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          language: '',\n        },\n      ],\n    });\n    const stream = await this.#room!.localParticipant!.streamText({\n      senderIdentity: participantIdentity,\n      topic: TOPIC_TRANSCRIPTION,\n      attributes: {\n        [ATTRIBUTE_TRANSCRIPTION_TRACK_ID]: trackSid,\n        [ATTRIBUTE_TRANSCRIPTION_FINAL]: isFinal.toString(),\n        [ATTRIBUTE_SEGMENT_ID]: segmentId,\n      },\n    });\n    await stream.write(text);\n    await stream.close();\n  }\n\n  #synthesizeAgentSpeech(\n    speechId: string,\n    source: string | LLMStream | AsyncIterable<string>,\n  ): SynthesisHandle {\n    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);\n    // TODO: where possible we would want to use deltas instead of full text segments, esp for LLM streams over the streamText API\n    synchronizer.on('textUpdated', async (text) => {\n      this.#agentTranscribedText = text.text;\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n      await this.#publishTranscription(\n        this.#room!.localParticipant!.identity!,\n        this.#agentPublication?.sid ?? '',\n        text.text,\n        text.final,\n        text.id,\n        this.#transcriptionId,\n      );\n      if (text.final) {\n        this.#transcriptionId = undefined;\n      }\n    });\n\n    if (!this.#agentOutput) {\n      throw new Error('agent output should be initialized when ready');\n    }\n\n    if (source instanceof LLMStream) {\n      source = llmStreamToStringIterable(speechId, source);\n    }\n\n    const ogSource = source;\n    if (!(typeof source === 'string')) {\n      // TODO(nbsp): itertools.tee\n    }\n\n    const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);\n    if (!ttsSource) {\n      throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');\n    }\n\n    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);\n  }\n\n  async #validateReplyIfPossible() {\n    if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {\n      this.#logger\n        .child({ speechId: this.#playingSpeech.id })\n        .debug('skipping validation, agent is speaking and does not allow interruptions');\n      return;\n    }\n\n    if (!this.#pendingAgentReply) {\n      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {\n        return;\n      }\n      this.#synthesizeAgentReply();\n    }\n\n    if (!this.#pendingAgentReply) {\n      throw new Error('pending agent reply is undefined');\n    }\n\n    // in some bad timimg, we could end up with two pushed agent replies inside the speech queue.\n    // so make sure we directly interrupt every reply when validating a new one\n    if (this.#speechQueueOpen.done) {\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        if (!speech.isReply) continue;\n        if (speech.allowInterruptions) speech.interrupt();\n      }\n    }\n\n    this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');\n\n    if (this.#lastSpeechTime) {\n      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;\n      const transcriptionDelay = Math.max(\n        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,\n        0,\n      );\n      const metrics: PipelineEOUMetrics = {\n        timestamp: Date.now(),\n        sequenceId: this.#pendingAgentReply.id,\n        endOfUtteranceDelay: timeSinceLastSpeech,\n        transcriptionDelay,\n      };\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    }\n\n    this.#addSpeechForPlayout(this.#pendingAgentReply);\n    this.#pendingAgentReply = undefined;\n    this.#transcribedInterimText = '';\n  }\n\n  #interruptIfPossible() {\n    if (\n      !this.#playingSpeech ||\n      !this.#playingSpeech.allowInterruptions ||\n      this.#playingSpeech.interrupted\n    ) {\n      return;\n    }\n\n    if (this.#opts.interruptMinWords !== 0) {\n      // check the final/interim transcribed text for the minimum word count\n      // to interrupt the agent speech\n      const interimWords = this.#opts.transcription.wordTokenizer.tokenize(\n        this.#transcribedInterimText,\n      );\n      if (interimWords.length < this.#opts.interruptMinWords) {\n        return;\n      }\n    }\n    this.#playingSpeech.interrupt();\n  }\n\n  #addSpeechForPlayout(handle: SpeechHandle) {\n    this.#speechQueue.put(handle);\n    this.#speechQueue.put(VoicePipelineAgent.FLUSH_SENTINEL);\n    this.#speechQueueOpen.resolve();\n  }\n\n  /** Close the voice assistant. */\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n\n    this.#room?.removeAllListeners(RoomEvent.ParticipantConnected);\n    // TODO(nbsp): await this.#deferredValidation.close()\n  }\n}\n\nasync function* llmStreamToStringIterable(\n  speechId: string,\n  stream: LLMStream,\n): AsyncIterable<string> {\n  const startTime = Date.now();\n  let firstFrame = true;\n  for await (const chunk of stream) {\n    const content = chunk.choices[0]?.delta.content;\n    if (!content) continue;\n\n    if (firstFrame) {\n      firstFrame = false;\n      log()\n        .child({ speechId, elapsed: Math.round(Date.now() - startTime) })\n        .debug('received first LLM token');\n    }\n    yield content;\n  }\n}\n\n/** This class is used to try to find the best time to validate the agent reply. */\nclass DeferredReplyValidation {\n  // if the STT gives us punctuation, we can try to validate the reply faster.\n  readonly PUNCTUATION = '.!?';\n  readonly PUNCTUATION_REDUCE_FACTOR = 0.75;\n  readonly LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech\n  readonly UNLIKELY_ENDPOINT_DELAY = 6000;\n\n  #validateFunc: () => Promise<void>;\n  #validatingPromise?: Promise<void>;\n  #validatingFuture = new Future();\n  #lastFinalTranscript = '';\n  #lastRecvEndOfSpeechTime = 0;\n  #speaking = false;\n  #endOfSpeechDelay: number;\n  #finalTranscriptDelay: number;\n  #turnDetector?: TurnDetector;\n  #agent: VoicePipelineAgent;\n  #abort?: AbortController;\n\n  constructor(\n    validateFunc: () => Promise<void>,\n    minEndpointingDelay: number,\n    agent: VoicePipelineAgent,\n    turnDetector?: TurnDetector,\n  ) {\n    this.#validateFunc = validateFunc;\n    this.#endOfSpeechDelay = minEndpointingDelay;\n    this.#finalTranscriptDelay = minEndpointingDelay;\n    this.#agent = agent;\n    this.#turnDetector = turnDetector;\n  }\n\n  get validating(): boolean {\n    return !this.#validatingFuture.done;\n  }\n\n  onHumanFinalTranscript(transcript: string) {\n    this.#lastFinalTranscript = transcript.trim();\n    if (this.#speaking) return;\n\n    const hasRecentEndOfSpeech =\n      Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;\n    let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;\n    delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;\n\n    this.#run(delay);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanStartOfSpeech(_: VADEvent) {\n    this.#speaking = true;\n    if (this.validating) {\n      this.#abort?.abort();\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanEndOfSpeech(_: VADEvent) {\n    this.#speaking = false;\n    this.#lastRecvEndOfSpeechTime = Date.now();\n\n    if (this.#lastFinalTranscript) {\n      const delay = this.#endWithPunctuation()\n        ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR\n        : 1_000;\n      this.#run(delay);\n    }\n  }\n\n  // TODO(nbsp): aclose\n\n  #endWithPunctuation(): boolean {\n    return (\n      this.#lastFinalTranscript.length > 0 &&\n      this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]!)\n    );\n  }\n\n  #resetStates() {\n    this.#lastFinalTranscript = '';\n    this.#lastRecvEndOfSpeechTime = 0;\n  }\n\n  #run(delay: number) {\n    const runTask = async (delay: number, chatCtx: ChatContext, signal: AbortSignal) => {\n      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {\n        const startTime = Date.now();\n        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);\n        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;\n        const elapsed = Date.now() - startTime;\n        if (eotProb < unlikelyThreshold) {\n          delay = this.UNLIKELY_ENDPOINT_DELAY;\n        }\n        delay = Math.max(0, delay - elapsed);\n      }\n      const timeout = setTimeout(() => {\n        this.#resetStates();\n        this.#validateFunc();\n      }, delay);\n      signal.addEventListener('abort', () => {\n        clearTimeout(timeout);\n      });\n    };\n\n    this.#abort?.abort();\n    this.#abort = new AbortController();\n    this.#validatingFuture = new Future();\n    const detectCtx = this.#agent.chatCtx.copy();\n    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });\n    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);\n  }\n}\n"],"mappings":"AASA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,kBAAkB;AAC3B,OAAO,kBAAkB;AACzB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAOP,SAAS,aAAa,aAAa,UAAU,UAAU,iBAAiB;AACxE,SAAS,WAAW;AAEpB,SAAmB,iBAAiB,kBAAkB,uBAAuB;AAC7E;AAAA,EACE,qBAAqB;AAAA,EACrB,iBAAiB;AAAA,EACjB;AAAA,OACK;AAEP,SAAS,uBAAuB,8BAA8B;AAE9D,SAAS,UAAU,iBAAiB,wBAAwB;AAC5D,SAAS,oBAAoB,oBAAoB,QAAQ,wBAAwB;AACjF,SAAkC,oBAAoB;AAEtD,SAAS,mBAAmB;AAC5B,SAAS,cAAc,yBAAyB;AAChD,SAAS,YAAY,uBAAuB;AAC5C,SAAS,oBAAoB;AAGtB,MAAM,wBAAwB;AACrC,IAAI;AACJ,IAAI;AAYG,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AAVU,SAAAA;AAAA,GAAA;AAgCL,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA,YAAY,oBAAI,IAAiB;AAAA,EACjC,qBAAoC,CAAC;AAAA,EACrC,OAAO;AAAA,EAEP,YAAY,OAA2B,WAAsB;AAC3D,SAAK,SAAS;AACd,SAAK,aAAa;AAClB,qBAAiB,WAAW;AAAA,EAC9B;AAAA,EAEA,OAAO,aAA+B;AACpC,WAAO,iBAAiB;AAAA,EAC1B;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAc,KAAa,OAAY;AACrC,SAAK,UAAU,IAAI,KAAK,KAAK;AAAA,EAC/B;AAAA,EAEA,YAAY,KAAa,YAAiB,QAAW;AACnD,WAAO,KAAK,UAAU,IAAI,GAAG,KAAK;AAAA,EACpC;AAAA,EAEA,IAAI,YAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,oBAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,oBAAoB,SAAsB;AACxC,SAAK,mBAAmB,KAAK,OAAO;AAAA,EACtC;AACF;AAEA,MAAM,2BAA8C,CAClD,OACA,YACc;AACd,SAAO,MAAM,IAAI,KAAK,EAAE,SAAS,QAAQ,MAAM,OAAO,CAAC;AACzD;AAEA,MAAM,2BAA8C,CAElD,GACA,SACmC;AACnC,SAAO;AACT;AA6BA,MAAM,mCAA8D;AAAA,EAClE,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,uBAAuB;AAAA,EAC9C,eAAe,IAAI,mBAAmB,KAAK;AAAA,EAC3C;AACF;AA2CA,MAAM,oBAAgC;AAAA,EACpC,SAAS,IAAI,YAAY;AAAA,EACzB,oBAAoB;AAAA,EACpB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,eAAe;AACjB;AAGO,MAAM,2BAA4B,aAAsD;AAAA;AAAA,EAEpF,6BAA6B;AAAA,EACtC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAElE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,qBAAqB,IAAI,OAAO;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA,kBAAkB;AAAA,EAClB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,OAAO;AAAA,EAC9B,eAAe,IAAI,mBAA4E;AAAA,EAC/F;AAAA,EACA,WAAW;AAAA,EACX;AAAA,EACA,eAAkD;AAAA,EAClD;AAAA,EACA,UAAU,IAAI;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,wBAAwB;AAAA,EAExB,YAEE,KAEA,KAEA,KAEA,KAEA,OAA4B,mBAC5B;AACA,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAE7C,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,iBAAiB,KAAK,GAAG;AAAA,IACrC;AAEA,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,iBAAiB,KAAK,IAAI,uBAAuB,CAAC;AAAA,IAC9D;AAEA,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AAEZ,SAAK,sBAAsB,IAAI;AAAA,MAC7B,KAAK,yBAAyB,KAAK,IAAI;AAAA,MACvC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK,MAAM;AAAA,IACb;AAAA,EACF;AAAA,EAEA,IAAI,SAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,OAAO,KAAsB;AAC/B,SAAK,MAAM,SAAS;AAAA,EACtB;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAEE,MAQA,cAAiD,MACjD;AACA,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,SAAK,KAAK,GAAG,gBAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,KAAK,GAAG,SAAS,mBAAmB,CAAC,YAAY;AACpD,UAAI,CAAC,WAAY;AACjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,YAAY,WAAW,WAAW,CAAC;AAAA,IACzF,CAAC;AAED,SAAK,KAAK,GAAG,SAAS,mBAAmB,CAAC,YAAY;AACpD,YAAM,aAAa,aAAa,WAAW,aAAa,iDAAgB;AACxE,UAAI,CAAC,WAAY;AAEjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,WAAW,CAAC;AAAA,IAClE,CAAC;AAED,SAAK,KAAK,GAAG,aAAa,mBAAmB,CAAC,YAAY;AACxD,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,GAAG,UAAU,sBAAsB,CAACC,iBAAmC;AAE1E,UAAI,KAAK,cAAc;AACrB;AAAA,MACF;AACA,WAAK,iBAAiB,KAAK,MAAMA,aAAY,QAAS;AAAA,IACxD,CAAC;AAED,SAAK,QAAQ;AACb,SAAK,eAAe;AAEpB,QAAI,aAAa;AACf,UAAI,OAAO,gBAAgB,UAAU;AACnC,aAAK,iBAAiB,WAAW;AAAA,MACnC,OAAO;AACL,aAAK,iBAAiB,YAAY,QAAS;AAAA,MAC7C;AAAA,IACF;AAEA,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA,EAGA,MAAM,IACJ,QACA,qBAAqB,MACrB,eAAe,MACQ;AACvB,UAAM,KAAK,mBAAmB;AAE9B,QAAI;AACJ,QAAI;AACJ,QAAI,cAAc;AAChB,oBAAc,iBAAiB,WAAW;AAC1C,UAAI,kBAAkB,WAAW;AAC/B,aAAK,QAAQ,KAAK,0DAA0D;AAAA,MAC9E,WAAW,OAAO,WAAW,UAAU;AACrC,oBAAY;AAAA,MACd,OAAO;AACL,oBAAY;AACZ,iBAAS,IAAI,mBAA2B;AAAA,MAC1C;AAAA,IACF;AAEA,UAAM,YAAY,aAAa,sBAAsB,oBAAoB,YAAY;AACrF,UAAM,kBAAkB,KAAK,uBAAuB,UAAU,IAAI,MAAM;AACxE,cAAU,WAAW,QAAQ,eAAe;AAE5C,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,sBAAsB;AACpE,WAAK,eAAe,gBAAgB,SAAS;AAAA,IAC/C,OAAO;AACL,WAAK,qBAAqB,SAAS;AAAA,IACrC;AAEA,QAAI,eAAe,WAAW;AAC5B,UAAI;AACJ,UAAI,OAAO,WAAW,UAAU;AAC9B,eAAO;AAAA,MACT,OAAO;AACL,eAAO;AACP,yBAAiB,SAAS,WAAW;AACnC,UAAC,OAAsC,IAAI,KAAK;AAChD,kBAAQ;AAAA,QACV;AACA,QAAC,OAAsC,MAAM;AAAA,MAC/C;AAEA,kBAAY,oBAAoB,YAAY,OAAO,EAAE,MAAM,MAAM,SAAS,UAAU,CAAC,CAAC;AACtF,WAAK,QAAQ,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,4CAA4C;AAAA,IACjF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmB,QAAQ,GAAG;AACzC,UAAM,UAAU,CAACC,WAA4C;AAC3D,aAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAzcpE;AA0cQ,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AACD,cAAM,IAAI,QAAQ,CAACC,aAAY,WAAWA,UAASD,MAAK,CAAC;AACzD,aAAI,UAAK,UAAL,mBAAY,aAAa;AAC3B,cAAI,CAAC,WAAW;AACd,oBAAM,UAAK,MAAM,qBAAX,mBAA6B,cAAc,EAAE,CAAC,qBAAqB,GAAG,MAAM;AAAA,UACpF;AAAA,QACF;AACA,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,kBAAkB;AACzB,WAAK,iBAAiB,OAAO;AAAA,IAC/B;AAEA,SAAK,mBAAmB,QAAQ,KAAK;AAAA,EACvC;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,eAAe,KAAK,MAAM,mBAAmB,IAAI,mBAAmB,KAAK;AAC9E,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,SAAK,cAAc,IAAI;AAAA,MACrB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK,MAAM;AAAA,IACb;AACA,SAAK,YAAY,GAAG,gBAAgB,iBAAiB,CAAC,UAAU;AAC9D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,qBAAqB,KAAK;AAAA,IACrD,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,oBAAoB,CAAC,UAAU;AACjE,UAAI,CAAC,KAAK,mBAAmB,MAAM;AACjC;AAAA,MACF;AACA,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,2BAA2B;AAAA,MAC7C;AAEA,UAAI,KAAK;AACT,UAAI,KAAK,MAAM,oBAAoB;AACjC,aAAK,KAAK,IAAI,GAAG,IAAI,MAAM,WAAW;AACtC,aAAK,aAAa,QAAQ,eAAe;AAAA,MAC3C;AAEA,UAAI,MAAM,kBAAkB,KAAK,MAAM,yBAAyB;AAC9D,aAAK,qBAAqB;AAAA,MAC5B;AAEA,UAAI,MAAM,uBAAuB,GAAG;AAClC,aAAK,kBAAkB,KAAK,IAAI,IAAI,MAAM;AAAA,MAC5C;AAAA,IACF,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,eAAe,CAAC,UAAU;AAC5D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,mBAAmB,KAAK;AAAA,IACnD,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,oBAAoB,OAAO,UAAU;AACvE,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,mBAAmB,WAAW;AAAA,MACrC;AACA,WAAK,0BAA0B,MAAM,aAAc,CAAC,EAAE;AAEtD,YAAM,KAAK;AAAA,QACT,KAAK,YAAa,YAAY;AAAA,QAC9B,KAAK,YAAa,gBAAiB;AAAA,QACnC,KAAK;AAAA,QACL;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF,CAAC;AACD,SAAK,YAAY,GAAG,gBAAgB,kBAAkB,OAAO,UAAU;AACrE,YAAM,gBAAgB,MAAM,aAAc,CAAC,EAAE;AAC7C,UAAI,CAAC,cAAe;AAEpB,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,mBAAmB,WAAW;AAAA,MACrC;AAEA,WAAK,2BAA2B,KAAK,IAAI;AACzC,WAAK,oBAAoB,KAAK,kBAAkB,MAAM,MAAM;AAE5D,YAAM,KAAK;AAAA,QACT,KAAK,YAAa,YAAY;AAAA,QAC9B,KAAK,YAAa,gBAAiB;AAAA,QACnC,KAAK;AAAA,QACL;AAAA,QACA,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAEA,WAAK,mBAAmB;AAExB,UACE,KAAK,MAAM,wBACV,CAAC,KAAK,kBAAkB,KAAK,eAAe,qBAC7C;AACA,aAAK,sBAAsB;AAAA,MAC7B;AAEA,WAAK,oBAAoB,uBAAuB,aAAa;AAE7D,YAAM,QAAQ,KAAK,MAAM,cAAc,cAAc,SAAS,aAAa;AAC3E,UAAI,MAAM,UAAU,GAAG;AAGrB,aAAK,qBAAqB;AAAA,MAC5B;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAO;AAvkBf;AAwkBI,SAAK,aAAa,cAAc;AAChC,UAAM,cAAc,IAAI,YAAY,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAC/E,UAAM,QAAQ,gBAAgB,iBAAiB,mBAAmB,WAAW;AAC7E,SAAK,oBAAoB,QAAM,gBAAK,UAAL,mBAAY,qBAAZ,mBAA8B;AAAA,MAC3D;AAAA,MACA,IAAI,oBAAoB,EAAE,QAAQ,YAAY,kBAAkB,CAAC;AAAA;AAGnE,UAAM,eAAe,IAAI,aAAa,WAAW;AACjD,SAAK,eAAe,IAAI,YAAY,cAAc,KAAK,IAAI;AAE3D,iBAAa,GAAG,kBAAkB,iBAAiB,MAAM;AACvD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,UAAU;AAAA,IAC9B,CAAC;AAED,iBAAa,GAAG,kBAAkB,iBAAiB,CAAC,MAAM;AACxD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,WAAW;AAAA,IAC/B,CAAC;AAED,SAAK,mBAAmB,QAAQ;AAEhC,WAAO,MAAM;AACX,YAAM,KAAK,iBAAiB;AAC5B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,aAAK,iBAAiB;AAAA,MACxB;AACA,WAAK,mBAAmB,IAAI,OAAO;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,wBAAwB;AA3mB1B;AA4mBI,eAAK,uBAAL,mBAAyB;AACzB,QAAI,KAAK,eAAe,KAAK,YAAY,UAAU;AACjD,WAAK,aAAa,YAAY,GAAG;AAAA,IACnC;AAEA,SAAK,qBAAqB,aAAa;AAAA,MACrC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK;AAAA,IACP;AACA,UAAM,YAAY,KAAK;AACvB,SAAK,kBAAkB,KAAK,sBAAsB,KAAK,iBAAiB,SAAS;AAAA,EACnF;AAAA,EAEA,sBACE,SACA,QAC0B;AAC1B,WAAO,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,UAAI,SAAS;AACX,cAAM,iBAAiB,OAAO;AAAA,MAChC;AAEA,YAAM,YAAY,KAAK,QAAQ,KAAK;AACpC,YAAM,gBAAgB,KAAK;AAC3B,UAAI,iBAAiB,cAAc,aAAa;AAC9C,aACG,CAAC,cAAc,gBAAgB,cAAc,kBAC9C,CAAC,cAAc,iBACf;AAGA,oBAAU,SAAS;AAAA,YACjB,YAAY,OAAO;AAAA,cACjB,MAAM,cAAc,gBAAgB;AAAA,cACpC,MAAM,SAAS;AAAA,YACjB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,gBAAU,SAAS;AAAA,QACjB,YAAY,OAAO;AAAA,UACjB,MAAM,iCAAQ;AAAA,UACd,MAAM,SAAS;AAAA,QACjB,CAAC;AAAA,MACH;AAEA,mBAAa,EAAE,YAAY,OAAQ,GAAG;AAEtC,UAAI;AACF,YAAI,UAAW,SAAQ;AACvB,YAAI,YAAY,MAAM,KAAK,MAAM,kBAAkB,MAAM,SAAS;AAClE,YAAI,cAAc,OAAO;AACvB,2CAAQ;AACR;AAAA,QACF;AAEA,YAAI,UAAW,SAAQ;AAEvB,YAAI,EAAE,qBAAqB,YAAY;AACrC,sBAAa,MAAM,yBAAyB,MAAM,SAAS;AAAA,QAC7D;AAEA,YAAI,OAAQ,aAAa;AACvB;AAAA,QACF;AAEA,cAAM,kBAAkB,KAAK,uBAAuB,OAAQ,IAAI,SAAS;AACzE,eAAQ,WAAW,WAAW,eAAe;AAAA,MAC/C,UAAE;AACA,yBAAiB;AACjB,qBAAa;AAAA,MACf;AACA,cAAQ;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAsB;AACtC,QAAI;AACF,YAAM,OAAO,sBAAsB;AAAA,IACrC,QAAQ;AACN;AAAA,IACF;AACA,UAAM,KAAK,kBAAmB,oBAAoB;AAClD,UAAM,kBAAkB,OAAO;AAC/B,QAAI,gBAAgB,YAAa;AAEjC,UAAM,eAAe,OAAO;AAC5B,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,UAAU,WAAW,KAAK;AAEhC,UAAM,6BAA6B,MAAM;AACvC,UAAI,CAAC,gBAAgB,gBAAgB,eAAe,OAAO,cAAe;AAC1E,YAAME,gBACJ,OAAO,kBAAkB,aAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AAKtE,UACE,OAAO,sBACP,CAACA,iBACD,WAAW,aAAa,KAAK,8BAC7B,CAAC,QAAQ,MACT;AACA;AAAA,MACF;AAEA,WAAK,QAAQ,MAAM,EAAE,gBAAgB,aAAa,CAAC,EAAE,MAAM,2BAA2B;AACtF,YAAM,UAAU,YAAY,OAAO,EAAE,MAAM,cAAc,MAAM,SAAS,KAAK,CAAC;AAC9E,WAAK,QAAQ,SAAS,KAAK,OAAO;AAClC,WAAK,KAAK,+BAAgC,OAAO;AAEjD,WAAK,kBAAkB,KAAK,gBAAgB,MAAM,aAAa,MAAM;AACrE,aAAO,kBAAkB;AAAA,IAC3B;AAGA,+BAA2B;AAE3B,WAAO,CAAC,QAAQ,MAAM;AACpB,YAAM,IAAI,QAAc,OAAO,YAAY;AACzC,mBAAW,SAAS,GAAG;AACvB,cAAM,QAAQ;AACd,gBAAQ;AAAA,MACV,CAAC;AACD,iCAA2B;AAC3B,UAAI,OAAO,YAAa;AAAA,IAC1B;AACA,+BAA2B;AAE3B,QAAI,gBAAgB,KAAK;AACzB,UAAM,eAAe,OAAO,kBAAkB,aAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AACzF,UAAM,cAAc,OAAO;AAE3B,QAAI,OAAO,iBAAiB,CAAC,gBAAgB,OAAO,gBAAgB;AAClE,UAAI,OAAO,oBAAoB;AAC7B,aAAK,QAAQ,SAAS,KAAK,GAAG,OAAO,kBAAkB;AAAA,MACzD;AACA,UAAI,aAAa;AACf,yBAAiB;AAAA,MACnB;AAEA,YAAM,MAAM,YAAY,OAAO,EAAE,MAAM,eAAe,MAAM,SAAS,UAAU,CAAC;AAChF,WAAK,QAAQ,SAAS,KAAK,GAAG;AAE9B,aAAO,oBAAoB;AAC3B,UAAI,aAAa;AACf,aAAK,KAAK,kCAAmC,GAAG;AAAA,MAClD,OAAO;AACL,aAAK,KAAK,gCAAiC,GAAG;AAAA,MAChD;AAEA,WAAK,QACF,MAAM;AAAA,QACL,iBAAiB;AAAA,QACjB;AAAA,QACA,UAAU,OAAO;AAAA,MACnB,CAAC,EACA,MAAM,wBAAwB;AAEjC,aAAO,QAAQ;AAAA,IACjB;AAEA,UAAM,uBAAuB,YAAY;AAGvC,UAAI,CAAC,gBAAgB,YAAa;AAElC,UAAI,OAAO,kBAAkB,KAAK,MAAM,mBAAmB;AACzD,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,IAAI,gBAAgB,OAAO,eAAe,CAAC,EACpE,KAAK,yCAAyC;AACjD;AAAA,MACF;AAEA,UAAI,gBAAgB,CAAC,OAAO,eAAe;AACzC,cAAM,IAAI,MAAM,2DAA2D;AAAA,MAC7E;AACA,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,UAAU;AAEnC,UAAI,iBAAiB,MAAM,SAAS;AAEpC,WAAK,KAAK,kCAAmC,gBAAgB;AAC7D,YAAM,cAAkC,CAAC;AACzC,iBAAW,QAAQ,kBAAkB;AACnC,cAAMC,QAAO,KAAK,KAAK,QAAQ,KAAK,MAAM,EAAE;AAAA,UAC1C,CAAC,YAAY,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,OAAO;AAAA,UACpE,CAAC,WAAW,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,MAAM;AAAA,QACpE;AACA,oBAAY,KAAK,EAAE,GAAG,MAAM,MAAAA,MAAK,CAAC;AAClC,aAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,uBAAuB;AAChC,YAAI;AACF,gBAAMA;AAAA,QACR,QAAQ;AACN,eAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,6BAA6B;AAAA,QACxC;AAAA,MACF;AAEA,YAAM,gBAAgB,CAAC;AACvB,YAAM,mBAAmB,CAAC;AAC1B,iBAAW,OAAO,aAAa;AAE7B,cAAMA,QAAO,MAAM,IAAI;AACvB,YAAI,CAACA,SAAQA,MAAK,WAAW,OAAW;AACxC,sBAAc,KAAK,GAAG;AACtB,yBAAiB,KAAK,YAAY,6BAA6BA,KAAI,CAAC;AAAA,MACtE;AAEA,UAAI,CAAC,cAAc,OAAQ;AAG3B,YAAM,qBAAqB,CAAC,YAAY,gBAAgB,eAAe,aAAa,CAAC;AACrF,yBAAmB,KAAK,GAAG,gBAAgB;AAG3C,YAAM,kBAAkB,aAAa;AAAA,QACnC,OAAO;AAAA,QACP,OAAO;AAAA,QACP,OAAO,iBAAiB;AAAA,QACxB;AAAA,MACF;AAGA,YAAM,UAAU,OAAO,OAAO,QAAQ,KAAK;AAC3C,cAAQ,SAAS,KAAK,GAAG,kBAAkB;AAC3C,cAAQ,SAAS,KAAK,GAAG,iBAAiB,WAAW,EAAE,iBAAiB;AAExE,YAAM,kBAAkB,KAAK,IAAI,KAAK;AAAA,QACpC;AAAA,QACA,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,kBAAkB,KAAK,uBAAuB,gBAAgB,IAAI,eAAe;AACvF,sBAAgB,WAAW,iBAAiB,eAAe;AAC3D,aAAO,gBAAgB,eAAe;AAEtC,WAAK,KAAK,iCAAkC,WAAW;AAAA,IACzD;AAEA,QAAI,WAAW;AACf,UAAM,OAAO,qBAAqB,EAAE,KAAK,MAAM;AAC7C,iBAAW;AAAA,IACb,CAAC;AACD,WAAO,CAAC,OAAO,sBAAsB;AACnC,YAAM,UAAU,OAAO,oBAAoB;AAC3C,YAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;AAClC,aAAO,OAAO,oBAAoB,QAAQ;AACxC,cAAM,SAAS,OAAO,oBAAoB,CAAC;AAC3C,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,eAAO,oBAAoB,MAAM;AACjC,aAAK,iBAAiB;AAAA,MACxB;AAEA,aAAO,oBAAoB,QAAQ,MAAM,OAAO,oBAAoB,IAAI,CAAC;AACzE,UAAI,UAAU;AACZ,eAAO,yBAAyB;AAAA,MAClC;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,MAAM,sBACJ,qBACA,UACA,MACA,SACA,IACA,WACA;AACA,SAAK,MAAO,iBAAkB,qBAAqB;AAAA,MACjD;AAAA,MACA;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF,CAAC;AACD,UAAM,SAAS,MAAM,KAAK,MAAO,iBAAkB,WAAW;AAAA,MAC5D,gBAAgB;AAAA,MAChB,OAAO;AAAA,MACP,YAAY;AAAA,QACV,CAAC,gCAAgC,GAAG;AAAA,QACpC,CAAC,6BAA6B,GAAG,QAAQ,SAAS;AAAA,QAClD,CAAC,oBAAoB,GAAG;AAAA,MAC1B;AAAA,IACF,CAAC;AACD,UAAM,OAAO,MAAM,IAAI;AACvB,UAAM,OAAO,MAAM;AAAA,EACrB;AAAA,EAEA,uBACE,UACA,QACiB;AACjB,UAAM,eAAe,IAAI,sBAAsB,sBAAsB;AAErE,iBAAa,GAAG,eAAe,OAAO,SAAS;AAv6BnD;AAw6BM,WAAK,wBAAwB,KAAK;AAClC,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,mBAAmB,WAAW;AAAA,MACrC;AACA,YAAM,KAAK;AAAA,QACT,KAAK,MAAO,iBAAkB;AAAA,UAC9B,UAAK,sBAAL,mBAAwB,QAAO;AAAA,QAC/B,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AACA,UAAI,KAAK,OAAO;AACd,aAAK,mBAAmB;AAAA,MAC1B;AAAA,IACF,CAAC;AAED,QAAI,CAAC,KAAK,cAAc;AACtB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAEA,QAAI,kBAAkB,WAAW;AAC/B,eAAS,0BAA0B,UAAU,MAAM;AAAA,IACrD;AAEA,UAAM,WAAW;AACjB,QAAI,EAAE,OAAO,WAAW,WAAW;AAAA,IAEnC;AAEA,UAAM,YAAY,KAAK,MAAM,kBAAkB,MAAM,QAAQ;AAC7D,QAAI,CAAC,WAAW;AACd,YAAM,IAAI,MAAM,+DAA+D;AAAA,IACjF;AAEA,WAAO,KAAK,aAAa,WAAW,UAAU,WAAW,YAAY;AAAA,EACvE;AAAA,EAEA,MAAM,2BAA2B;AAC/B,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,oBAAoB;AAClE,WAAK,QACF,MAAM,EAAE,UAAU,KAAK,eAAe,GAAG,CAAC,EAC1C,MAAM,yEAAyE;AAClF;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,UAAI,KAAK,MAAM,uBAAuB,CAAC,KAAK,iBAAiB;AAC3D;AAAA,MACF;AACA,WAAK,sBAAsB;AAAA,IAC7B;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAIA,QAAI,KAAK,iBAAiB,MAAM;AAC9B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,YAAI,CAAC,OAAO,QAAS;AACrB,YAAI,OAAO,mBAAoB,QAAO,UAAU;AAAA,MAClD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,UAAU,KAAK,mBAAmB,GAAG,CAAC,EAAE,MAAM,uBAAuB;AAE1F,QAAI,KAAK,iBAAiB;AACxB,YAAM,sBAAsB,KAAK,IAAI,IAAI,KAAK;AAC9C,YAAM,qBAAqB,KAAK;AAAA,SAC7B,KAAK,4BAA4B,KAAK,KAAK;AAAA,QAC5C;AAAA,MACF;AACA,YAAM,UAA8B;AAAA,QAClC,WAAW,KAAK,IAAI;AAAA,QACpB,YAAY,KAAK,mBAAmB;AAAA,QACpC,qBAAqB;AAAA,QACrB;AAAA,MACF;AACA,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C;AAEA,SAAK,qBAAqB,KAAK,kBAAkB;AACjD,SAAK,qBAAqB;AAC1B,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,uBAAuB;AACrB,QACE,CAAC,KAAK,kBACN,CAAC,KAAK,eAAe,sBACrB,KAAK,eAAe,aACpB;AACA;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,sBAAsB,GAAG;AAGtC,YAAM,eAAe,KAAK,MAAM,cAAc,cAAc;AAAA,QAC1D,KAAK;AAAA,MACP;AACA,UAAI,aAAa,SAAS,KAAK,MAAM,mBAAmB;AACtD;AAAA,MACF;AAAA,IACF;AACA,SAAK,eAAe,UAAU;AAAA,EAChC;AAAA,EAEA,qBAAqB,QAAsB;AACzC,SAAK,aAAa,IAAI,MAAM;AAC5B,SAAK,aAAa,IAAI,mBAAmB,cAAc;AACvD,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA;AAAA,EAGA,MAAM,QAAQ;AA9hChB;AA+hCI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AAEA,eAAK,UAAL,mBAAY,mBAAmB,UAAU;AAAA,EAE3C;AACF;AAEA,gBAAgB,0BACd,UACA,QACuB;AA3iCzB;AA4iCE,QAAM,YAAY,KAAK,IAAI;AAC3B,MAAI,aAAa;AACjB,mBAAiB,SAAS,QAAQ;AAChC,UAAM,WAAU,WAAM,QAAQ,CAAC,MAAf,mBAAkB,MAAM;AACxC,QAAI,CAAC,QAAS;AAEd,QAAI,YAAY;AACd,mBAAa;AACb,UAAI,EACD,MAAM,EAAE,UAAU,SAAS,KAAK,MAAM,KAAK,IAAI,IAAI,SAAS,EAAE,CAAC,EAC/D,MAAM,0BAA0B;AAAA,IACrC;AACA,UAAM;AAAA,EACR;AACF;AAGA,MAAM,wBAAwB;AAAA;AAAA,EAEnB,cAAc;AAAA,EACd,4BAA4B;AAAA,EAC5B,4BAA4B;AAAA;AAAA,EAC5B,0BAA0B;AAAA,EAEnC;AAAA,EACA;AAAA,EACA,oBAAoB,IAAI,OAAO;AAAA,EAC/B,uBAAuB;AAAA,EACvB,2BAA2B;AAAA,EAC3B,YAAY;AAAA,EACZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,cACA,qBACA,OACA,cACA;AACA,SAAK,gBAAgB;AACrB,SAAK,oBAAoB;AACzB,SAAK,wBAAwB;AAC7B,SAAK,SAAS;AACd,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,CAAC,KAAK,kBAAkB;AAAA,EACjC;AAAA,EAEA,uBAAuB,YAAoB;AACzC,SAAK,uBAAuB,WAAW,KAAK;AAC5C,QAAI,KAAK,UAAW;AAEpB,UAAM,uBACJ,KAAK,IAAI,IAAI,KAAK,2BAA2B,KAAK;AACpD,QAAI,QAAQ,uBAAuB,KAAK,oBAAoB,KAAK;AACjE,YAAQ,KAAK,oBAAoB,IAAI,QAAQ,KAAK,4BAA4B;AAE9E,SAAK,KAAK,KAAK;AAAA,EACjB;AAAA;AAAA,EAGA,qBAAqB,GAAa;AA9mCpC;AA+mCI,SAAK,YAAY;AACjB,QAAI,KAAK,YAAY;AACnB,iBAAK,WAAL,mBAAa;AAAA,IACf;AAAA,EACF;AAAA;AAAA,EAGA,mBAAmB,GAAa;AAC9B,SAAK,YAAY;AACjB,SAAK,2BAA2B,KAAK,IAAI;AAEzC,QAAI,KAAK,sBAAsB;AAC7B,YAAM,QAAQ,KAAK,oBAAoB,IACnC,KAAK,oBAAoB,KAAK,4BAC9B;AACJ,WAAK,KAAK,KAAK;AAAA,IACjB;AAAA,EACF;AAAA;AAAA,EAIA,sBAA+B;AAC7B,WACE,KAAK,qBAAqB,SAAS,KACnC,KAAK,YAAY,SAAS,KAAK,qBAAqB,KAAK,qBAAqB,SAAS,CAAC,CAAE;AAAA,EAE9F;AAAA,EAEA,eAAe;AACb,SAAK,uBAAuB;AAC5B,SAAK,2BAA2B;AAAA,EAClC;AAAA,EAEA,KAAK,OAAe;AAhpCtB;AAipCI,UAAM,UAAU,OAAOH,QAAe,SAAsB,WAAwB;AAClF,UAAI,KAAK,wBAAwB,CAAC,KAAK,aAAa,KAAK,eAAe;AACtE,cAAM,YAAY,KAAK,IAAI;AAC3B,cAAM,UAAU,MAAM,KAAK,cAAc,iBAAiB,OAAO;AACjE,cAAM,oBAAoB,KAAK,cAAc;AAC7C,cAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAI,UAAU,mBAAmB;AAC/B,UAAAA,SAAQ,KAAK;AAAA,QACf;AACA,QAAAA,SAAQ,KAAK,IAAI,GAAGA,SAAQ,OAAO;AAAA,MACrC;AACA,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,aAAa;AAClB,aAAK,cAAc;AAAA,MACrB,GAAGA,MAAK;AACR,aAAO,iBAAiB,SAAS,MAAM;AACrC,qBAAa,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAEA,eAAK,WAAL,mBAAa;AACb,SAAK,SAAS,IAAI,gBAAgB;AAClC,SAAK,oBAAoB,IAAI,OAAO;AACpC,UAAM,YAAY,KAAK,OAAO,QAAQ,KAAK;AAC3C,cAAU,OAAO,EAAE,MAAM,KAAK,OAAO,iBAAiB,MAAM,SAAS,KAAK,CAAC;AAC3E,SAAK,qBAAqB,QAAQ,OAAO,WAAW,KAAK,OAAO,MAAM;AAAA,EACxE;AACF;","names":["VPAEvent","participant","delay","resolve","isUsingTools","task"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "0.7.8",
+  "version": "0.7.9",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

package/src/constants.ts

@@ -4,4 +4,5 @@
 export const ATTRIBUTE_TRANSCRIPTION_TRACK_ID = 'lk.transcribed_track_id';
 export const ATTRIBUTE_TRANSCRIPTION_FINAL = 'lk.transcription_final';
 export const TOPIC_TRANSCRIPTION = 'lk.transcription';
+export const ATTRIBUTE_SEGMENT_ID = 'lk.segment_id';
 export const TOPIC_CHAT = 'lk.chat';

package/src/multimodal/multimodal_agent.ts

@@ -18,9 +18,11 @@ import {
   TrackPublishOptions,
   TrackSource,
 } from '@livekit/rtc-node';
+import { randomUUID } from 'node:crypto';
 import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
 import {
+  ATTRIBUTE_SEGMENT_ID,
   ATTRIBUTE_TRANSCRIPTION_FINAL,
   ATTRIBUTE_TRANSCRIPTION_TRACK_ID,
   TOPIC_TRANSCRIPTION,
@@ -72,6 +74,7 @@ export class MultimodalAgent extends EventEmitter {
 
   #textResponseRetries = 0;
   #maxTextResponseRetries: number;
+  #transcriptionId?: string;
 
   constructor({
     model,
@@ -257,13 +260,20 @@ export class MultimodalAgent extends EventEmitter {
 
         const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
         synchronizer.on('textUpdated', async (text) => {
+          if (!this.#transcriptionId) {
+            this.#transcriptionId = randomUUID();
+          }
           await this.#publishTranscription(
             this.room!.localParticipant!.identity!,
             this.#getLocalTrackSid()!,
             text.text,
             text.final,
             text.id,
+            this.#transcriptionId,
           );
+          if (text.final) {
+            this.#transcriptionId = undefined;
+          }
         });
 
         const handle = this.#agentPlayout?.play(
@@ -312,7 +322,17 @@ export class MultimodalAgent extends EventEmitter {
         const participantIdentity = this.linkedParticipant?.identity;
         const trackSid = this.subscribedTrack?.sid;
         if (participantIdentity && trackSid) {
-          await this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);
+          if (!this.#transcriptionId) {
+            this.#transcriptionId = randomUUID();
+          }
+          await this.#publishTranscription(
+            participantIdentity,
+            trackSid,
+            '…',
+            false,
+            ev.itemId,
+            this.#transcriptionId,
+          );
         } else {
           this.#logger.error('Participant or track not set');
         }
@@ -325,13 +345,18 @@ export class MultimodalAgent extends EventEmitter {
         const participantIdentity = this.linkedParticipant?.identity;
         const trackSid = this.subscribedTrack?.sid;
         if (participantIdentity && trackSid) {
+          if (!this.#transcriptionId) {
+            this.#transcriptionId = randomUUID();
+          }
           await this.#publishTranscription(
             participantIdentity,
             trackSid,
             transcription,
             true,
             ev.itemId,
+            this.#transcriptionId,
           );
+          this.#transcriptionId = undefined;
         } else {
           this.#logger.error('Participant or track not set');
         }
@@ -360,7 +385,17 @@ export class MultimodalAgent extends EventEmitter {
         const participantIdentity = this.linkedParticipant?.identity;
         const trackSid = this.subscribedTrack?.sid;
         if (participantIdentity && trackSid) {
-          await this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);
+          if (!this.#transcriptionId) {
+            this.#transcriptionId = randomUUID();
+          }
+          await this.#publishTranscription(
+            participantIdentity,
+            trackSid,
+            '…',
+            false,
+            ev.itemId,
+            this.#transcriptionId,
+          );
         }
       });
 
@@ -492,6 +527,7 @@ export class MultimodalAgent extends EventEmitter {
     text: string,
     isFinal: boolean,
     id: string,
+    segmentId: string,
   ): Promise<void> {
     this.#logger.debug(
       `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,
@@ -522,6 +558,7 @@ export class MultimodalAgent extends EventEmitter {
       attributes: {
         [ATTRIBUTE_TRANSCRIPTION_TRACK_ID]: trackSid,
         [ATTRIBUTE_TRANSCRIPTION_FINAL]: isFinal.toString(),
+        [ATTRIBUTE_SEGMENT_ID]: segmentId,
       },
     });
     await stream.write(text);

package/src/pipeline/pipeline_agent.ts

@@ -18,6 +18,7 @@ import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
 import {
+  ATTRIBUTE_SEGMENT_ID,
   ATTRIBUTE_TRANSCRIPTION_FINAL,
   ATTRIBUTE_TRANSCRIPTION_TRACK_ID,
   TOPIC_TRANSCRIPTION,
@@ -537,6 +538,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         this.#transcribedInterimText,
         false,
         this.#transcriptionId,
+        this.#transcriptionId,
       );
     });
     this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, async (event) => {
@@ -556,6 +558,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         this.transcribedText,
         true,
         this.#transcriptionId,
+        this.#transcriptionId,
       );
 
       this.#transcriptionId = undefined;
@@ -895,6 +898,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     text: string,
     isFinal: boolean,
     id: string,
+    segmentId: string,
   ) {
     this.#room!.localParticipant!.publishTranscription({
       participantIdentity: participantIdentity,
@@ -916,6 +920,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       attributes: {
         [ATTRIBUTE_TRANSCRIPTION_TRACK_ID]: trackSid,
         [ATTRIBUTE_TRANSCRIPTION_FINAL]: isFinal.toString(),
+        [ATTRIBUTE_SEGMENT_ID]: segmentId,
       },
     });
     await stream.write(text);
@@ -930,13 +935,20 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     // TODO: where possible we would want to use deltas instead of full text segments, esp for LLM streams over the streamText API
     synchronizer.on('textUpdated', async (text) => {
       this.#agentTranscribedText = text.text;
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
       await this.#publishTranscription(
         this.#room!.localParticipant!.identity!,
         this.#agentPublication?.sid ?? '',
         text.text,
         text.final,
         text.id,
+        this.#transcriptionId,
       );
+      if (text.final) {
+        this.#transcriptionId = undefined;
+      }
     });
 
     if (!this.#agentOutput) {