@livekit/agents 0.6.4 → 0.7.1

package/dist/pipeline/pipeline_agent.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/pipeline_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { LocalTrackPublication, RemoteParticipant, Room } from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport EventEmitter from 'node:events';\nimport type {\n  CallableFunctionResult,\n  FunctionCallInfo,\n  FunctionContext,\n  LLM,\n} from '../llm/index.js';\nimport { LLMEvent, LLMStream } from '../llm/index.js';\nimport { ChatContext, ChatMessage, ChatRole } from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { AgentMetrics, PipelineEOUMetrics } from '../metrics/base.js';\nimport { type STT, StreamAdapter as STTStreamAdapter, SpeechEventType } from '../stt/index.js';\nimport {\n  SentenceTokenizer as BasicSentenceTokenizer,\n  WordTokenizer as BasicWordTokenizer,\n  hyphenateWord,\n} from '../tokenize/basic/index.js';\nimport type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';\nimport type { TTS } from '../tts/index.js';\nimport { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { SpeechSource, SynthesisHandle } from './agent_output.js';\nimport { AgentOutput } from './agent_output.js';\nimport { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';\nimport { HumanInput, HumanInputEvent } from './human_input.js';\nimport { SpeechHandle } from './speech_handle.js';\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\nlet speechData: { sequenceId: string } | undefined;\n\nexport type BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n) => LLMStream | false | void | Promise<LLMStream | false | void>;\n\nexport type BeforeTTSCallback = (\n  agent: VoicePipelineAgent,\n  source: string | AsyncIterable<string>,\n) => SpeechSource;\n\nexport enum VPAEvent {\n  USER_STARTED_SPEAKING,\n  USER_STOPPED_SPEAKING,\n  AGENT_STARTED_SPEAKING,\n  AGENT_STOPPED_SPEAKING,\n  USER_SPEECH_COMMITTED,\n  AGENT_SPEECH_COMMITTED,\n  AGENT_SPEECH_INTERRUPTED,\n  FUNCTION_CALLS_COLLECTED,\n  FUNCTION_CALLS_FINISHED,\n  METRICS_COLLECTED,\n}\n\nexport type VPACallbacks = {\n  [VPAEvent.USER_STARTED_SPEAKING]: () => void;\n  [VPAEvent.USER_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STARTED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.USER_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;\n  [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;\n  [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;\n  [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;\n};\n\nexport class AgentCallContext {\n  #agent: VoicePipelineAgent;\n  #llmStream: LLMStream;\n  #metadata = new Map<string, any>();\n  #extraChatMessages: ChatMessage[] = [];\n  static #current: AgentCallContext;\n\n  constructor(agent: VoicePipelineAgent, llmStream: LLMStream) {\n    this.#agent = agent;\n    this.#llmStream = llmStream;\n    AgentCallContext.#current = this;\n  }\n\n  static getCurrent(): AgentCallContext {\n    return AgentCallContext.#current;\n  }\n\n  get agent(): VoicePipelineAgent {\n    return this.#agent;\n  }\n\n  storeMetadata(key: string, value: any) {\n    this.#metadata.set(key, value);\n  }\n\n  getMetadata(key: string, orDefault: any = undefined) {\n    return this.#metadata.get(key) || orDefault;\n  }\n\n  get llmStream(): LLMStream {\n    return this.#llmStream;\n  }\n\n  get extraChatMessages() {\n    return this.#extraChatMessages;\n  }\n\n  addExtraChatMessage(message: ChatMessage) {\n    this.#extraChatMessages.push(message);\n  }\n}\n\nconst defaultBeforeLLMCallback: BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n): LLMStream => {\n  return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });\n};\n\nconst defaultBeforeTTSCallback: BeforeTTSCallback = (\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  _: VoicePipelineAgent,\n  text: string | AsyncIterable<string>,\n): string | AsyncIterable<string> => {\n  return text;\n};\n\nexport interface AgentTranscriptionOptions {\n  /** Whether to forward the user transcription to the client */\n  userTranscription: boolean;\n  /** Whether to forward the agent transcription to the client */\n  agentTranscription: boolean;\n  /**\n   * The speed at which the agent's speech transcription is forwarded to the client.\n   * We try to mimic the agent's speech speed by adjusting the transcription speed.\n   */\n  agentTranscriptionSpeech: number;\n  /**\n   * The tokenizer used to split the speech into sentences.\n   * This is used to decide when to mark a transcript as final for the agent transcription.\n   */\n  sentenceTokenizer: SentenceTokenizer;\n  /**\n   * The tokenizer used to split the speech into words.\n   * This is used to simulate the \"interim results\" of the agent transcription.\n   */\n  wordTokenizer: WordTokenizer;\n  /**\n   * A function that takes a string (word) as input and returns a list of strings,\n   * representing the hyphenated parts of the word.\n   */\n  hyphenateWord: (word: string) => string[];\n}\n\nconst defaultAgentTranscriptionOptions: AgentTranscriptionOptions = {\n  userTranscription: true,\n  agentTranscription: true,\n  agentTranscriptionSpeech: 1,\n  sentenceTokenizer: new BasicSentenceTokenizer(),\n  wordTokenizer: new BasicWordTokenizer(false),\n  hyphenateWord: hyphenateWord,\n};\n\nexport interface VPAOptions {\n  /** Chat context for the assistant. */\n  chatCtx?: ChatContext;\n  /** Function context for the assistant. */\n  fncCtx?: FunctionContext;\n  /** Whether to allow the user to interrupt the assistant. */\n  allowInterruptions: boolean;\n  /** Minimum duration of speech to consider for interruption. */\n  interruptSpeechDuration: number;\n  /** Minimum number of words to consider for interuption. This may increase latency. */\n  interruptMinWords: number;\n  /** Delay to wait before considering the user speech done. */\n  minEndpointingDelay: number;\n  maxNestedFncCalls: number;\n  /* Whether to preemptively synthesize responses. */\n  preemptiveSynthesis: boolean;\n  /*\n   * Callback called when the assistant is about to synthesize a reply.\n   *\n   * @remarks\n   * Returning void will create a default LLM stream.\n   * You can also return your own LLM stream by calling `llm.chat()`.\n   * Returning `false` ill cancel the synthesis of the reply.\n   */\n  beforeLLMCallback: BeforeLLMCallback;\n  /*\n   * Callback called when the assistant is about to synthesize speech.\n   *\n   * @remarks\n   * This can be used to customize text before synthesis\n   * (e.g. editing the pronunciation of a word).\n   */\n  beforeTTSCallback: BeforeTTSCallback;\n  /** Options for assistant transcription. */\n  transcription: AgentTranscriptionOptions;\n}\n\nconst defaultVPAOptions: VPAOptions = {\n  chatCtx: new ChatContext(),\n  allowInterruptions: true,\n  interruptSpeechDuration: 50,\n  interruptMinWords: 0,\n  minEndpointingDelay: 500,\n  maxNestedFncCalls: 1,\n  preemptiveSynthesis: false,\n  beforeLLMCallback: defaultBeforeLLMCallback,\n  beforeTTSCallback: defaultBeforeTTSCallback,\n  transcription: defaultAgentTranscriptionOptions,\n};\n\n/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */\nexport class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<VPACallbacks>) {\n  /** Minimum time played for the user speech to be committed to the chat context. */\n  readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n\n  #vad: VAD;\n  #stt: STT;\n  #llm: LLM;\n  #tts: TTS;\n  #opts: VPAOptions;\n  #humanInput?: HumanInput;\n  #agentOutput?: AgentOutput;\n  #trackPublishedFut = new Future();\n  #pendingAgentReply?: SpeechHandle;\n  #agentReplyTask?: CancellablePromise<void>;\n  #playingSpeech?: SpeechHandle;\n  #transcribedText = '';\n  #transcribedInterimText = '';\n  #speechQueueOpen = new Future();\n  #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();\n  #updateStateTask?: CancellablePromise<void>;\n  #started = false;\n  #room?: Room;\n  #participant: RemoteParticipant | string | null = null;\n  #deferredValidation: DeferredReplyValidation;\n  #logger = log();\n  #agentPublication?: LocalTrackPublication;\n  #lastFinalTranscriptTime?: number;\n  #lastSpeechTime?: number;\n\n  constructor(\n    /** Voice Activity Detection instance. */\n    vad: VAD,\n    /** Speech-to-Text instance. */\n    stt: STT,\n    /** Large Language Model instance. */\n    llm: LLM,\n    /** Text-to-Speech instance. */\n    tts: TTS,\n    /** Additional VoicePipelineAgent options. */\n    opts: Partial<VPAOptions> = defaultVPAOptions,\n  ) {\n    super();\n\n    this.#opts = { ...defaultVPAOptions, ...opts };\n\n    if (!stt.capabilities.streaming) {\n      stt = new STTStreamAdapter(stt, vad);\n    }\n\n    if (!tts.capabilities.streaming) {\n      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());\n    }\n\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#llm = llm;\n    this.#tts = tts;\n\n    this.#deferredValidation = new DeferredReplyValidation(\n      this.#validateReplyIfPossible.bind(this),\n      this.#opts.minEndpointingDelay,\n    );\n  }\n\n  get fncCtx(): FunctionContext | undefined {\n    return this.#opts.fncCtx;\n  }\n\n  set fncCtx(ctx: FunctionContext) {\n    this.#opts.fncCtx = ctx;\n  }\n\n  get chatCtx(): ChatContext {\n    return this.#opts.chatCtx!;\n  }\n\n  get llm(): LLM {\n    return this.#llm;\n  }\n\n  get tts(): TTS {\n    return this.#tts;\n  }\n\n  get stt(): STT {\n    return this.#stt;\n  }\n\n  get vad(): VAD {\n    return this.#vad;\n  }\n\n  /** Start the voice assistant. */\n  start(\n    /** The room to connect to. */\n    room: Room,\n    /**\n     * The participant to listen to.\n     *\n     * @remarks\n     * Can be a participant or an identity.\n     * If omitted, the first participant in the room will be selected.\n     */\n    participant: RemoteParticipant | string | null = null,\n  ) {\n    if (this.#started) {\n      throw new Error('voice assistant already started');\n    }\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {\n      if (!speechData) return;\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });\n    });\n\n    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {\n      if (!speechData) return;\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });\n    });\n\n    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n      // automatically link to the first participant that connects, if not already linked\n      if (this.#participant) {\n        return;\n      }\n      this.#linkParticipant.call(this, participant.identity!);\n    });\n\n    this.#room = room;\n    this.#participant = participant;\n\n    if (participant) {\n      if (typeof participant === 'string') {\n        this.#linkParticipant(participant);\n      } else {\n        this.#linkParticipant(participant.identity!);\n      }\n    }\n\n    this.#run();\n  }\n\n  /** Play a speech source through the voice assistant. */\n  async say(\n    source: string | LLMStream | AsyncIterable<string>,\n    allowInterruptions = true,\n    addToChatCtx = true,\n  ): Promise<SpeechHandle> {\n    await this.#trackPublishedFut.await;\n\n    let callContext: AgentCallContext | undefined;\n    let fncSource: string | AsyncIterable<string> | undefined;\n    if (addToChatCtx) {\n      callContext = AgentCallContext.getCurrent();\n      if (source instanceof LLMStream) {\n        this.#logger.warn('LLMStream will be ignored for function call chat context');\n      } else if (typeof source === 'string') {\n        fncSource = source;\n      } else {\n        fncSource = source;\n        source = new AsyncIterableQueue<string>();\n      }\n    }\n\n    const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);\n    const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);\n    newHandle.initialize(source, synthesisHandle);\n\n    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {\n      this.#playingSpeech.addNestedSpeech(newHandle);\n    } else {\n      this.#addSpeechForPlayout(newHandle);\n    }\n\n    if (callContext && fncSource) {\n      let text: string;\n      if (typeof source === 'string') {\n        text = fncSource as string;\n      } else {\n        text = '';\n        for await (const chunk of fncSource) {\n          (source as AsyncIterableQueue<string>).put(chunk);\n          text += chunk;\n        }\n        (source as AsyncIterableQueue<string>).close();\n      }\n\n      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));\n      this.#logger.child({ text }).debug('added speech to function call chat context');\n    }\n\n    return newHandle;\n  }\n\n  #updateState(state: AgentState, delay = 0) {\n    const runTask = (delay: number): CancellablePromise<void> => {\n      return new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n        await new Promise((resolve) => setTimeout(resolve, delay));\n        if (this.#room?.isConnected) {\n          if (!cancelled) {\n            await this.#room.localParticipant?.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state });\n          }\n        }\n        resolve();\n      });\n    };\n\n    if (this.#updateStateTask) {\n      this.#updateStateTask.cancel();\n    }\n\n    this.#updateStateTask = runTask(delay);\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.#room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.#participant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    this.#humanInput = new HumanInput(this.#room, this.#vad, this.#stt, this.#participant);\n    this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STARTED_SPEAKING);\n      this.#deferredValidation.onHumanStartOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {\n      if (!this.#trackPublishedFut.done) {\n        return;\n      }\n      if (!this.#agentOutput) {\n        throw new Error('agent output is undefined');\n      }\n\n      let tv = 1;\n      if (this.#opts.allowInterruptions) {\n        tv = Math.max(0, 1 - event.probability);\n        this.#agentOutput.playout.targetVolume = tv;\n      }\n\n      if (event.speechDuration >= this.#opts.interruptSpeechDuration) {\n        this.#interruptIfPossible();\n      }\n\n      if (event.rawAccumulatedSpeech > 0) {\n        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;\n      }\n    });\n    this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STOPPED_SPEAKING);\n      this.#deferredValidation.onHumanEndOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {\n      this.#transcribedInterimText = event.alternatives![0].text;\n    });\n    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {\n      const newTranscript = event.alternatives![0].text;\n      if (!newTranscript) return;\n\n      this.#lastFinalTranscriptTime = Date.now();\n      this.#transcribedText += (this.#transcribedText ? ' ' : '') + newTranscript;\n\n      if (\n        this.#opts.preemptiveSynthesis &&\n        (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)\n      ) {\n        this.#synthesizeAgentReply();\n      }\n\n      this.#deferredValidation.onHumanFinalTranscript(newTranscript);\n\n      const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);\n      if (words.length >= 3) {\n        // VAD can sometimes not detect that the human is speaking.\n        // to make the interruption more reliable, we also interrupt on the final transcript.\n        this.#interruptIfPossible();\n      }\n    });\n  }\n\n  async #run() {\n    this.#updateState('initializing');\n    const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);\n    const track = LocalAudioTrack.createAudioTrack('assistant_voice', audioSource);\n    this.#agentPublication = await this.#room?.localParticipant?.publishTrack(\n      track,\n      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n    );\n\n    const agentPlayout = new AgentPlayout(audioSource);\n    this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);\n\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {\n      this.emit(VPAEvent.AGENT_STARTED_SPEAKING);\n      this.#updateState('speaking');\n    });\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {\n      this.emit(VPAEvent.AGENT_STOPPED_SPEAKING);\n      this.#updateState('listening');\n    });\n\n    this.#trackPublishedFut.resolve();\n\n    while (true) {\n      await this.#speechQueueOpen.await;\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        this.#playingSpeech = undefined;\n      }\n      this.#speechQueueOpen = new Future();\n    }\n  }\n\n  #synthesizeAgentReply() {\n    this.#pendingAgentReply?.cancel();\n    if (this.#humanInput && this.#humanInput.speaking) {\n      this.#updateState('thinking', 200);\n    }\n\n    this.#pendingAgentReply = SpeechHandle.createAssistantReply(\n      this.#opts.allowInterruptions,\n      true,\n      this.#transcribedText,\n    );\n    const newHandle = this.#pendingAgentReply;\n    this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);\n  }\n\n  #synthesizeAnswerTask(\n    oldTask: CancellablePromise<void> | undefined,\n    handle?: SpeechHandle,\n  ): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      const copiedCtx = this.chatCtx.copy();\n      const playingSpeech = this.#playingSpeech;\n      if (playingSpeech && playingSpeech.initialized) {\n        if (\n          (!playingSpeech.userQuestion || playingSpeech.userCommitted) &&\n          !playingSpeech.speechCommitted\n        ) {\n          // the speech is playing but not committed yet,\n          // add it to the chat context for this new reply synthesis\n          copiedCtx.messages.push(\n            ChatMessage.create({\n              text: playingSpeech.synthesisHandle.text,\n              role: ChatRole.ASSISTANT,\n            }),\n          );\n        }\n      }\n\n      copiedCtx.messages.push(\n        ChatMessage.create({\n          text: handle?.userQuestion,\n          role: ChatRole.USER,\n        }),\n      );\n\n      speechData = { sequenceId: handle!.id };\n\n      try {\n        if (cancelled) resolve();\n        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);\n        if (llmStream === false) {\n          handle?.cancel();\n          return;\n        }\n\n        if (cancelled) resolve();\n        // fallback to default impl if no custom/user stream is returned\n        if (!(llmStream instanceof LLMStream)) {\n          llmStream = (await defaultBeforeLLMCallback(this, copiedCtx)) as LLMStream;\n        }\n\n        if (handle!.interrupted) {\n          return;\n        }\n\n        const synthesisHandle = this.#synthesizeAgentSpeech(handle!.id, llmStream);\n        handle!.initialize(llmStream, synthesisHandle);\n      } finally {\n        speechData = undefined;\n      }\n      resolve();\n    });\n  }\n\n  async #playSpeech(handle: SpeechHandle) {\n    try {\n      await handle.waitForInitialization();\n    } catch {\n      return;\n    }\n    await this.#agentPublication!.waitForSubscription();\n    const synthesisHandle = handle.synthesisHandle;\n    if (synthesisHandle.interrupted) return;\n\n    const userQuestion = handle.userQuestion;\n    const playHandle = synthesisHandle.play();\n    const joinFut = playHandle.join();\n\n    const commitUserQuestionIfNeeded = () => {\n      if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted) return;\n      const isUsingTools =\n        handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n\n      // make sure at least some speech was played before committing the user message\n      // since we try to validate as fast as possible it is possible the agent gets interrupted\n      // really quickly (barely audible), we don't want to mark this question as \"answered\".\n      if (\n        handle.allowInterruptions &&\n        !isUsingTools &&\n        playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT &&\n        !joinFut.done\n      ) {\n        return;\n      }\n\n      this.#logger.child({ userTranscript: userQuestion }).debug('committed user transcript');\n      const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });\n      this.chatCtx.messages.push(userMsg);\n      this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);\n\n      this.#transcribedText = this.#transcribedText.slice(userQuestion.length);\n      handle.markUserCommitted();\n    };\n\n    // wait for the playHandle to finish and check every 1s if user question should be committed\n    commitUserQuestionIfNeeded();\n\n    while (!joinFut.done) {\n      await new Promise<void>(async (resolve) => {\n        setTimeout(resolve, 500);\n        await joinFut.await;\n        resolve();\n      });\n      commitUserQuestionIfNeeded();\n      if (handle.interrupted) break;\n    }\n    commitUserQuestionIfNeeded();\n\n    const collectedText = handle.synthesisHandle.text;\n    const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n    const interrupted = handle.interrupted;\n\n    if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {\n      if (handle.extraToolsMessages) {\n        this.chatCtx.messages.push(...handle.extraToolsMessages);\n      }\n      if (interrupted) {\n        collectedText + '…';\n      }\n\n      const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });\n      this.chatCtx.messages.push(msg);\n\n      handle.markSpeechCommitted();\n      if (interrupted) {\n        this.emit(VPAEvent.AGENT_SPEECH_INTERRUPTED, msg);\n      } else {\n        this.emit(VPAEvent.AGENT_SPEECH_COMMITTED, msg);\n      }\n\n      this.#logger\n        .child({\n          agentTranscript: collectedText,\n          interrupted,\n          speechId: handle.id,\n        })\n        .debug('committed agent speech');\n\n      handle.setDone();\n    }\n\n    const executeFunctionCalls = async () => {\n      // if the answer is using tools, execute the functions and automatically generate\n      // a response to the user question from the returned values\n      if (!isUsingTools || interrupted) return;\n\n      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {\n        this.#logger\n          .child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth })\n          .warn('max function calls nested depth reached');\n        return;\n      }\n\n      if (userQuestion && !handle.userCommitted) {\n        throw new Error('user speech should have been committed before using tools');\n      }\n      const llmStream = handle.source;\n      const newFunctionCalls = llmStream.functionCalls;\n\n      new AgentCallContext(this, llmStream);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);\n      const calledFuncs: FunctionCallInfo[] = [];\n      for (const func of newFunctionCalls) {\n        const task = func.func.execute(func.params).then(\n          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),\n          (error) => ({ name: func.name, toolCallId: func.toolCallId, error }),\n        );\n        calledFuncs.push({ ...func, task });\n        this.#logger\n          .child({ function: func.name, speechId: handle.id })\n          .debug('executing AI function');\n        try {\n          await task;\n        } catch {\n          this.#logger\n            .child({ function: func.name, speechId: handle.id })\n            .error('error executing AI function');\n        }\n      }\n\n      const toolCallsInfo = [];\n      const toolCallsResults = [];\n      for (const fnc of calledFuncs) {\n        // ignore the function calls that return void\n        const task = await fnc.task;\n        if (!task || task.result === undefined) continue;\n        toolCallsInfo.push(fnc);\n        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));\n      }\n\n      if (!toolCallsInfo.length) return;\n\n      // generate an answer from the tool calls\n      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];\n      extraToolsMessages.push(...toolCallsResults);\n\n      // create a nested speech handle\n      const newSpeechHandle = SpeechHandle.createToolSpeech(\n        handle.allowInterruptions,\n        handle.addToChatCtx,\n        handle.fncNestedDepth + 1,\n        extraToolsMessages,\n      );\n\n      // synthesize the tool speech with the chat ctx from llmStream\n      const chatCtx = handle.source.chatCtx.copy();\n      chatCtx.messages.push(...extraToolsMessages);\n      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);\n\n      const answerLLMStream = this.llm.chat({\n        chatCtx,\n        fncCtx: this.fncCtx,\n      });\n      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);\n      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);\n      handle.addNestedSpeech(newSpeechHandle);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);\n    };\n\n    let finished = false;\n    const task = executeFunctionCalls().then(() => {\n      finished = true;\n    });\n    while (!handle.nestedSpeechFinished) {\n      const changed = handle.nestedSpeechChanged();\n      await Promise.race([changed, task]);\n      while (handle.nestedSpeechHandles.length) {\n        const speech = handle.nestedSpeechHandles[0]!;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        handle.nestedSpeechHandles.shift();\n        this.#playingSpeech = handle;\n      }\n\n      handle.nestedSpeechHandles.forEach(() => handle.nestedSpeechHandles.pop());\n      if (finished) {\n        handle.markNestedSpeechFinished();\n      }\n    }\n    handle.setDone();\n  }\n\n  #synthesizeAgentSpeech(\n    speechId: string,\n    source: string | LLMStream | AsyncIterable<string>,\n  ): SynthesisHandle {\n    if (!this.#agentOutput) {\n      throw new Error('agent output should be initialized when ready');\n    }\n\n    if (source instanceof LLMStream) {\n      source = llmStreamToStringIterable(speechId, source);\n    }\n\n    const ogSource = source;\n    if (!(typeof source === 'string')) {\n      // TODO(nbsp): itertools.tee\n    }\n\n    const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);\n    if (!ttsSource) {\n      throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');\n    }\n\n    return this.#agentOutput.synthesize(speechId, ttsSource);\n  }\n\n  async #validateReplyIfPossible() {\n    if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {\n      this.#logger\n        .child({ speechId: this.#playingSpeech.id })\n        .debug('skipping validation, agent is speaking and does not allow interruptions');\n      return;\n    }\n\n    if (!this.#pendingAgentReply) {\n      if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {\n        return;\n      }\n      this.#synthesizeAgentReply();\n    }\n\n    if (!this.#pendingAgentReply) {\n      throw new Error('pending agent reply is undefined');\n    }\n\n    // in some bad timimg, we could end up with two pushed agent replies inside the speech queue.\n    // so make sure we directly interrupt every reply when validating a new one\n    if (this.#speechQueueOpen.done) {\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        if (!speech.isReply) continue;\n        if (speech.allowInterruptions) speech.interrupt();\n      }\n    }\n\n    this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');\n\n    if (this.#lastSpeechTime) {\n      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;\n      const transcriptionDelay = Math.max(\n        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,\n        0,\n      );\n      const metrics: PipelineEOUMetrics = {\n        timestamp: Date.now(),\n        sequenceId: this.#pendingAgentReply.id,\n        endOfUtteranceDelay: timeSinceLastSpeech,\n        transcriptionDelay,\n      };\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    }\n\n    this.#addSpeechForPlayout(this.#pendingAgentReply);\n    this.#pendingAgentReply = undefined;\n    this.#transcribedInterimText = '';\n  }\n\n  #interruptIfPossible() {\n    if (\n      !this.#playingSpeech ||\n      !this.#playingSpeech.allowInterruptions ||\n      this.#playingSpeech.interrupted\n    ) {\n      return;\n    }\n\n    if (this.#opts.interruptMinWords !== 0) {\n      // check the final/interim transcribed text for the minimum word count\n      // to interrupt the agent speech\n      const interimWords = this.#opts.transcription.wordTokenizer.tokenize(\n        this.#transcribedInterimText,\n      );\n      if (interimWords.length < this.#opts.interruptMinWords) {\n        return;\n      }\n    }\n    this.#playingSpeech.interrupt();\n  }\n\n  #addSpeechForPlayout(handle: SpeechHandle) {\n    this.#speechQueue.put(handle);\n    this.#speechQueue.put(VoicePipelineAgent.FLUSH_SENTINEL);\n    this.#speechQueueOpen.resolve();\n  }\n\n  /** Close the voice assistant. */\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n\n    this.#room?.removeAllListeners(RoomEvent.ParticipantConnected);\n    // TODO(nbsp): await this.#deferredValidation.close()\n  }\n}\n\nasync function* llmStreamToStringIterable(\n  speechId: string,\n  stream: LLMStream,\n): AsyncIterable<string> {\n  const startTime = Date.now();\n  let firstFrame = true;\n  for await (const chunk of stream) {\n    const content = chunk.choices[0]?.delta.content;\n    if (!content) continue;\n\n    if (firstFrame) {\n      firstFrame = false;\n      log()\n        .child({ speechId, elapsed: Math.round(Date.now() - startTime) })\n        .debug('received first LLM token');\n    }\n    yield content;\n  }\n}\n\n/** This class is used to try to find the best time to validate the agent reply. */\nclass DeferredReplyValidation {\n  // if the STT gives us punctuation, we can try to validate the reply faster.\n  readonly PUNCTUATION = '.!?';\n  readonly PUNCTUATION_REDUCE_FACTOR = 0.75;\n  readonly LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech\n\n  #validateFunc: () => Promise<void>;\n  #validatingPromise?: Promise<void>;\n  #validatingFuture = new Future();\n  #lastFinalTranscript = '';\n  #lastRecvEndOfSpeechTime = 0;\n  #speaking = false;\n  #endOfSpeechDelay: number;\n  #finalTranscriptDelay: number;\n  #abort?: AbortController;\n\n  constructor(validateFunc: () => Promise<void>, minEndpointingDelay: number) {\n    this.#validateFunc = validateFunc;\n    this.#endOfSpeechDelay = minEndpointingDelay;\n    this.#finalTranscriptDelay = minEndpointingDelay;\n  }\n\n  get validating(): boolean {\n    return !this.#validatingFuture.done;\n  }\n\n  onHumanFinalTranscript(transcript: string) {\n    this.#lastFinalTranscript = transcript.trim();\n    if (this.#speaking) return;\n\n    const hasRecentEndOfSpeech =\n      Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;\n    let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;\n    delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;\n\n    this.#run(delay);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanStartOfSpeech(_: VADEvent) {\n    this.#speaking = true;\n    if (this.validating) {\n      this.#abort?.abort();\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanEndOfSpeech(_: VADEvent) {\n    this.#speaking = false;\n    this.#lastRecvEndOfSpeechTime = Date.now();\n\n    if (this.#lastFinalTranscript) {\n      const delay = this.#endWithPunctuation()\n        ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR\n        : 1_000;\n      this.#run(delay);\n    }\n  }\n\n  // TODO(nbsp): aclose\n\n  #endWithPunctuation(): boolean {\n    return (\n      this.#lastFinalTranscript.length > 0 &&\n      this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]!)\n    );\n  }\n\n  #resetStates() {\n    this.#lastFinalTranscript = '';\n    this.#lastRecvEndOfSpeechTime = 0;\n  }\n\n  #run(delay: number) {\n    const runTask = async (delay: number, signal: AbortSignal) => {\n      const timeout = setTimeout(() => {\n        this.#resetStates();\n        this.#validateFunc();\n      }, delay);\n      signal.addEventListener('abort', () => {\n        clearTimeout(timeout);\n      });\n    };\n\n    this.#abort?.abort();\n    this.#abort = new AbortController();\n    this.#validatingFuture = new Future();\n    this.#validatingPromise = runTask(delay, this.#abort.signal);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAMO;AAEP,yBAAyB;AAOzB,iBAAoC;AACpC,IAAAA,cAAmD;AACnD,iBAAoB;AAEpB,iBAA6E;AAC7E,mBAIO;AAGP,iBAA4D;AAC5D,mBAAiF;AACjF,iBAAsD;AAEtD,0BAA4B;AAC5B,2BAAgD;AAChD,yBAA4C;AAC5C,2BAA6B;AAGtB,MAAM,wBAAwB;AACrC,IAAI;AAYG,IAAK,WAAL,kBAAKC,cAAL;AACL,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AAVU,SAAAA;AAAA,GAAA;AA0BL,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA,YAAY,oBAAI,IAAiB;AAAA,EACjC,qBAAoC,CAAC;AAAA,EACrC,OAAO;AAAA,EAEP,YAAY,OAA2B,WAAsB;AAC3D,SAAK,SAAS;AACd,SAAK,aAAa;AAClB,qBAAiB,WAAW;AAAA,EAC9B;AAAA,EAEA,OAAO,aAA+B;AACpC,WAAO,iBAAiB;AAAA,EAC1B;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAc,KAAa,OAAY;AACrC,SAAK,UAAU,IAAI,KAAK,KAAK;AAAA,EAC/B;AAAA,EAEA,YAAY,KAAa,YAAiB,QAAW;AACnD,WAAO,KAAK,UAAU,IAAI,GAAG,KAAK;AAAA,EACpC;AAAA,EAEA,IAAI,YAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,oBAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,oBAAoB,SAAsB;AACxC,SAAK,mBAAmB,KAAK,OAAO;AAAA,EACtC;AACF;AAEA,MAAM,2BAA8C,CAClD,OACA,YACc;AACd,SAAO,MAAM,IAAI,KAAK,EAAE,SAAS,QAAQ,MAAM,OAAO,CAAC;AACzD;AAEA,MAAM,2BAA8C,CAElD,GACA,SACmC;AACnC,SAAO;AACT;AA6BA,MAAM,mCAA8D;AAAA,EAClE,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,aAAAC,kBAAuB;AAAA,EAC9C,eAAe,IAAI,aAAAC,cAAmB,KAAK;AAAA,EAC3C,eAAe;AACjB;AAuCA,MAAM,oBAAgC;AAAA,EACpC,SAAS,IAAI,wBAAY;AAAA,EACzB,oBAAoB;AAAA,EACpB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,eAAe;AACjB;AAGO,MAAM,2BAA4B,mBAAAC,QAAsD;AAAA;AAAA,EAEpF,6BAA6B;AAAA,EACtC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAElE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,qBAAqB,IAAI,oBAAO;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA,mBAAmB;AAAA,EACnB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,oBAAO;AAAA,EAC9B,eAAe,IAAI,gCAA4E;AAAA,EAC/F;AAAA,EACA,WAAW;AAAA,EACX;AAAA,EACA,eAAkD;AAAA,EAClD;AAAA,EACA,cAAU,gBAAI;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAEE,KAEA,KAEA,KAEA,KAEA,OAA4B,mBAC5B;AACA,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAE7C,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,WAAAC,cAAiB,KAAK,GAAG;AAAA,IACrC;AAEA,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,WAAAC,cAAiB,KAAK,IAAI,aAAAJ,kBAAuB,CAAC;AAAA,IAC9D;AAEA,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AAEZ,SAAK,sBAAsB,IAAI;AAAA,MAC7B,KAAK,yBAAyB,KAAK,IAAI;AAAA,MACvC,KAAK,MAAM;AAAA,IACb;AAAA,EACF;AAAA,EAEA,IAAI,SAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,OAAO,KAAsB;AAC/B,SAAK,MAAM,SAAS;AAAA,EACtB;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAEE,MAQA,cAAiD,MACjD;AACA,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,SAAK,KAAK,GAAG,2BAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,KAAK,GAAG,oBAAS,mBAAmB,CAAC,YAAY;AACpD,UAAI,CAAC,WAAY;AACjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,YAAY,WAAW,WAAW,CAAC;AAAA,IACzF,CAAC;AAED,SAAK,KAAK,GAAG,oBAAS,mBAAmB,CAAC,YAAY;AACpD,UAAI,CAAC,WAAY;AACjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,YAAY,WAAW,WAAW,CAAC;AAAA,IACzF,CAAC;AAED,SAAK,KAAK,GAAG,wBAAa,mBAAmB,CAAC,YAAY;AACxD,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,GAAG,0BAAU,sBAAsB,CAACK,iBAAmC;AAE1E,UAAI,KAAK,cAAc;AACrB;AAAA,MACF;AACA,WAAK,iBAAiB,KAAK,MAAMA,aAAY,QAAS;AAAA,IACxD,CAAC;AAED,SAAK,QAAQ;AACb,SAAK,eAAe;AAEpB,QAAI,aAAa;AACf,UAAI,OAAO,gBAAgB,UAAU;AACnC,aAAK,iBAAiB,WAAW;AAAA,MACnC,OAAO;AACL,aAAK,iBAAiB,YAAY,QAAS;AAAA,MAC7C;AAAA,IACF;AAEA,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA,EAGA,MAAM,IACJ,QACA,qBAAqB,MACrB,eAAe,MACQ;AACvB,UAAM,KAAK,mBAAmB;AAE9B,QAAI;AACJ,QAAI;AACJ,QAAI,cAAc;AAChB,oBAAc,iBAAiB,WAAW;AAC1C,UAAI,kBAAkB,sBAAW;AAC/B,aAAK,QAAQ,KAAK,0DAA0D;AAAA,MAC9E,WAAW,OAAO,WAAW,UAAU;AACrC,oBAAY;AAAA,MACd,OAAO;AACL,oBAAY;AACZ,iBAAS,IAAI,gCAA2B;AAAA,MAC1C;AAAA,IACF;AAEA,UAAM,YAAY,kCAAa,sBAAsB,oBAAoB,YAAY;AACrF,UAAM,kBAAkB,KAAK,uBAAuB,UAAU,IAAI,MAAM;AACxE,cAAU,WAAW,QAAQ,eAAe;AAE5C,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,sBAAsB;AACpE,WAAK,eAAe,gBAAgB,SAAS;AAAA,IAC/C,OAAO;AACL,WAAK,qBAAqB,SAAS;AAAA,IACrC;AAEA,QAAI,eAAe,WAAW;AAC5B,UAAI;AACJ,UAAI,OAAO,WAAW,UAAU;AAC9B,eAAO;AAAA,MACT,OAAO;AACL,eAAO;AACP,yBAAiB,SAAS,WAAW;AACnC,UAAC,OAAsC,IAAI,KAAK;AAChD,kBAAQ;AAAA,QACV;AACA,QAAC,OAAsC,MAAM;AAAA,MAC/C;AAEA,kBAAY,oBAAoB,wBAAY,OAAO,EAAE,MAAM,MAAM,qBAAS,UAAU,CAAC,CAAC;AACtF,WAAK,QAAQ,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,4CAA4C;AAAA,IACjF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmB,QAAQ,GAAG;AACzC,UAAM,UAAU,CAACC,WAA4C;AAC3D,aAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AA5apE;AA6aQ,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AACD,cAAM,IAAI,QAAQ,CAACC,aAAY,WAAWA,UAASD,MAAK,CAAC;AACzD,aAAI,UAAK,UAAL,mBAAY,aAAa;AAC3B,cAAI,CAAC,WAAW;AACd,oBAAM,UAAK,MAAM,qBAAX,mBAA6B,cAAc,EAAE,CAAC,qBAAqB,GAAG,MAAM;AAAA,UACpF;AAAA,QACF;AACA,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,kBAAkB;AACzB,WAAK,iBAAiB,OAAO;AAAA,IAC/B;AAEA,SAAK,mBAAmB,QAAQ,KAAK;AAAA,EACvC;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,eAAe,KAAK,MAAM,mBAAmB,IAAI,mBAAmB,KAAK;AAC9E,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,8BAAW,KAAK,OAAO,KAAK,MAAM,KAAK,MAAM,KAAK,YAAY;AACrF,SAAK,YAAY,GAAG,mCAAgB,iBAAiB,CAAC,UAAU;AAC9D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,qBAAqB,KAAK;AAAA,IACrD,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,oBAAoB,CAAC,UAAU;AACjE,UAAI,CAAC,KAAK,mBAAmB,MAAM;AACjC;AAAA,MACF;AACA,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,2BAA2B;AAAA,MAC7C;AAEA,UAAI,KAAK;AACT,UAAI,KAAK,MAAM,oBAAoB;AACjC,aAAK,KAAK,IAAI,GAAG,IAAI,MAAM,WAAW;AACtC,aAAK,aAAa,QAAQ,eAAe;AAAA,MAC3C;AAEA,UAAI,MAAM,kBAAkB,KAAK,MAAM,yBAAyB;AAC9D,aAAK,qBAAqB;AAAA,MAC5B;AAEA,UAAI,MAAM,uBAAuB,GAAG;AAClC,aAAK,kBAAkB,KAAK,IAAI,IAAI,MAAM;AAAA,MAC5C;AAAA,IACF,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,eAAe,CAAC,UAAU;AAC5D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,mBAAmB,KAAK;AAAA,IACnD,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,oBAAoB,CAAC,UAAU;AACjE,WAAK,0BAA0B,MAAM,aAAc,CAAC,EAAE;AAAA,IACxD,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,kBAAkB,CAAC,UAAU;AAC/D,YAAM,gBAAgB,MAAM,aAAc,CAAC,EAAE;AAC7C,UAAI,CAAC,cAAe;AAEpB,WAAK,2BAA2B,KAAK,IAAI;AACzC,WAAK,qBAAqB,KAAK,mBAAmB,MAAM,MAAM;AAE9D,UACE,KAAK,MAAM,wBACV,CAAC,KAAK,kBAAkB,KAAK,eAAe,qBAC7C;AACA,aAAK,sBAAsB;AAAA,MAC7B;AAEA,WAAK,oBAAoB,uBAAuB,aAAa;AAE7D,YAAM,QAAQ,KAAK,MAAM,cAAc,cAAc,SAAS,aAAa;AAC3E,UAAI,MAAM,UAAU,GAAG;AAGrB,aAAK,qBAAqB;AAAA,MAC5B;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAO;AAzgBf;AA0gBI,SAAK,aAAa,cAAc;AAChC,UAAM,cAAc,IAAI,4BAAY,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAC/E,UAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,WAAW;AAC7E,SAAK,oBAAoB,QAAM,gBAAK,UAAL,mBAAY,qBAAZ,mBAA8B;AAAA,MAC3D;AAAA,MACA,IAAI,oCAAoB,EAAE,QAAQ,4BAAY,kBAAkB,CAAC;AAAA;AAGnE,UAAM,eAAe,IAAI,kCAAa,WAAW;AACjD,SAAK,eAAe,IAAI,gCAAY,cAAc,KAAK,IAAI;AAE3D,iBAAa,GAAG,uCAAkB,iBAAiB,MAAM;AACvD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,UAAU;AAAA,IAC9B,CAAC;AAED,iBAAa,GAAG,uCAAkB,iBAAiB,CAAC,MAAM;AACxD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,WAAW;AAAA,IAC/B,CAAC;AAED,SAAK,mBAAmB,QAAQ;AAEhC,WAAO,MAAM;AACX,YAAM,KAAK,iBAAiB;AAC5B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,aAAK,iBAAiB;AAAA,MACxB;AACA,WAAK,mBAAmB,IAAI,oBAAO;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,wBAAwB;AA7iB1B;AA8iBI,eAAK,uBAAL,mBAAyB;AACzB,QAAI,KAAK,eAAe,KAAK,YAAY,UAAU;AACjD,WAAK,aAAa,YAAY,GAAG;AAAA,IACnC;AAEA,SAAK,qBAAqB,kCAAa;AAAA,MACrC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK;AAAA,IACP;AACA,UAAM,YAAY,KAAK;AACvB,SAAK,kBAAkB,KAAK,sBAAsB,KAAK,iBAAiB,SAAS;AAAA,EACnF;AAAA,EAEA,sBACE,SACA,QAC0B;AAC1B,WAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,UAAI,SAAS;AACX,kBAAM,+BAAiB,OAAO;AAAA,MAChC;AAEA,YAAM,YAAY,KAAK,QAAQ,KAAK;AACpC,YAAM,gBAAgB,KAAK;AAC3B,UAAI,iBAAiB,cAAc,aAAa;AAC9C,aACG,CAAC,cAAc,gBAAgB,cAAc,kBAC9C,CAAC,cAAc,iBACf;AAGA,oBAAU,SAAS;AAAA,YACjB,wBAAY,OAAO;AAAA,cACjB,MAAM,cAAc,gBAAgB;AAAA,cACpC,MAAM,qBAAS;AAAA,YACjB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,gBAAU,SAAS;AAAA,QACjB,wBAAY,OAAO;AAAA,UACjB,MAAM,iCAAQ;AAAA,UACd,MAAM,qBAAS;AAAA,QACjB,CAAC;AAAA,MACH;AAEA,mBAAa,EAAE,YAAY,OAAQ,GAAG;AAEtC,UAAI;AACF,YAAI,UAAW,SAAQ;AACvB,YAAI,YAAY,MAAM,KAAK,MAAM,kBAAkB,MAAM,SAAS;AAClE,YAAI,cAAc,OAAO;AACvB,2CAAQ;AACR;AAAA,QACF;AAEA,YAAI,UAAW,SAAQ;AAEvB,YAAI,EAAE,qBAAqB,uBAAY;AACrC,sBAAa,MAAM,yBAAyB,MAAM,SAAS;AAAA,QAC7D;AAEA,YAAI,OAAQ,aAAa;AACvB;AAAA,QACF;AAEA,cAAM,kBAAkB,KAAK,uBAAuB,OAAQ,IAAI,SAAS;AACzE,eAAQ,WAAW,WAAW,eAAe;AAAA,MAC/C,UAAE;AACA,qBAAa;AAAA,MACf;AACA,cAAQ;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAsB;AACtC,QAAI;AACF,YAAM,OAAO,sBAAsB;AAAA,IACrC,QAAQ;AACN;AAAA,IACF;AACA,UAAM,KAAK,kBAAmB,oBAAoB;AAClD,UAAM,kBAAkB,OAAO;AAC/B,QAAI,gBAAgB,YAAa;AAEjC,UAAM,eAAe,OAAO;AAC5B,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,UAAU,WAAW,KAAK;AAEhC,UAAM,6BAA6B,MAAM;AACvC,UAAI,CAAC,gBAAgB,gBAAgB,eAAe,OAAO,cAAe;AAC1E,YAAME,gBACJ,OAAO,kBAAkB,wBAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AAKtE,UACE,OAAO,sBACP,CAACA,iBACD,WAAW,aAAa,KAAK,8BAC7B,CAAC,QAAQ,MACT;AACA;AAAA,MACF;AAEA,WAAK,QAAQ,MAAM,EAAE,gBAAgB,aAAa,CAAC,EAAE,MAAM,2BAA2B;AACtF,YAAM,UAAU,wBAAY,OAAO,EAAE,MAAM,cAAc,MAAM,qBAAS,KAAK,CAAC;AAC9E,WAAK,QAAQ,SAAS,KAAK,OAAO;AAClC,WAAK,KAAK,+BAAgC,OAAO;AAEjD,WAAK,mBAAmB,KAAK,iBAAiB,MAAM,aAAa,MAAM;AACvE,aAAO,kBAAkB;AAAA,IAC3B;AAGA,+BAA2B;AAE3B,WAAO,CAAC,QAAQ,MAAM;AACpB,YAAM,IAAI,QAAc,OAAO,YAAY;AACzC,mBAAW,SAAS,GAAG;AACvB,cAAM,QAAQ;AACd,gBAAQ;AAAA,MACV,CAAC;AACD,iCAA2B;AAC3B,UAAI,OAAO,YAAa;AAAA,IAC1B;AACA,+BAA2B;AAE3B,UAAM,gBAAgB,OAAO,gBAAgB;AAC7C,UAAM,eAAe,OAAO,kBAAkB,wBAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AACzF,UAAM,cAAc,OAAO;AAE3B,QAAI,OAAO,iBAAiB,CAAC,gBAAgB,OAAO,gBAAgB;AAClE,UAAI,OAAO,oBAAoB;AAC7B,aAAK,QAAQ,SAAS,KAAK,GAAG,OAAO,kBAAkB;AAAA,MACzD;AACA,UAAI,aAAa;AACf,wBAAgB;AAAA,MAClB;AAEA,YAAM,MAAM,wBAAY,OAAO,EAAE,MAAM,eAAe,MAAM,qBAAS,UAAU,CAAC;AAChF,WAAK,QAAQ,SAAS,KAAK,GAAG;AAE9B,aAAO,oBAAoB;AAC3B,UAAI,aAAa;AACf,aAAK,KAAK,kCAAmC,GAAG;AAAA,MAClD,OAAO;AACL,aAAK,KAAK,gCAAiC,GAAG;AAAA,MAChD;AAEA,WAAK,QACF,MAAM;AAAA,QACL,iBAAiB;AAAA,QACjB;AAAA,QACA,UAAU,OAAO;AAAA,MACnB,CAAC,EACA,MAAM,wBAAwB;AAEjC,aAAO,QAAQ;AAAA,IACjB;AAEA,UAAM,uBAAuB,YAAY;AAGvC,UAAI,CAAC,gBAAgB,YAAa;AAElC,UAAI,OAAO,kBAAkB,KAAK,MAAM,mBAAmB;AACzD,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,IAAI,gBAAgB,OAAO,eAAe,CAAC,EACpE,KAAK,yCAAyC;AACjD;AAAA,MACF;AAEA,UAAI,gBAAgB,CAAC,OAAO,eAAe;AACzC,cAAM,IAAI,MAAM,2DAA2D;AAAA,MAC7E;AACA,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,UAAU;AAEnC,UAAI,iBAAiB,MAAM,SAAS;AAEpC,WAAK,KAAK,kCAAmC,gBAAgB;AAC7D,YAAM,cAAkC,CAAC;AACzC,iBAAW,QAAQ,kBAAkB;AACnC,cAAMC,QAAO,KAAK,KAAK,QAAQ,KAAK,MAAM,EAAE;AAAA,UAC1C,CAAC,YAAY,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,OAAO;AAAA,UACpE,CAAC,WAAW,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,MAAM;AAAA,QACpE;AACA,oBAAY,KAAK,EAAE,GAAG,MAAM,MAAAA,MAAK,CAAC;AAClC,aAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,uBAAuB;AAChC,YAAI;AACF,gBAAMA;AAAA,QACR,QAAQ;AACN,eAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,6BAA6B;AAAA,QACxC;AAAA,MACF;AAEA,YAAM,gBAAgB,CAAC;AACvB,YAAM,mBAAmB,CAAC;AAC1B,iBAAW,OAAO,aAAa;AAE7B,cAAMA,QAAO,MAAM,IAAI;AACvB,YAAI,CAACA,SAAQA,MAAK,WAAW,OAAW;AACxC,sBAAc,KAAK,GAAG;AACtB,yBAAiB,KAAK,wBAAY,6BAA6BA,KAAI,CAAC;AAAA,MACtE;AAEA,UAAI,CAAC,cAAc,OAAQ;AAG3B,YAAM,qBAAqB,CAAC,wBAAY,gBAAgB,eAAe,aAAa,CAAC;AACrF,yBAAmB,KAAK,GAAG,gBAAgB;AAG3C,YAAM,kBAAkB,kCAAa;AAAA,QACnC,OAAO;AAAA,QACP,OAAO;AAAA,QACP,OAAO,iBAAiB;AAAA,QACxB;AAAA,MACF;AAGA,YAAM,UAAU,OAAO,OAAO,QAAQ,KAAK;AAC3C,cAAQ,SAAS,KAAK,GAAG,kBAAkB;AAC3C,cAAQ,SAAS,KAAK,GAAG,iBAAiB,WAAW,EAAE,iBAAiB;AAExE,YAAM,kBAAkB,KAAK,IAAI,KAAK;AAAA,QACpC;AAAA,QACA,QAAQ,KAAK;AAAA,MACf,CAAC;AACD,YAAM,kBAAkB,KAAK,uBAAuB,gBAAgB,IAAI,eAAe;AACvF,sBAAgB,WAAW,iBAAiB,eAAe;AAC3D,aAAO,gBAAgB,eAAe;AAEtC,WAAK,KAAK,iCAAkC,WAAW;AAAA,IACzD;AAEA,QAAI,WAAW;AACf,UAAM,OAAO,qBAAqB,EAAE,KAAK,MAAM;AAC7C,iBAAW;AAAA,IACb,CAAC;AACD,WAAO,CAAC,OAAO,sBAAsB;AACnC,YAAM,UAAU,OAAO,oBAAoB;AAC3C,YAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;AAClC,aAAO,OAAO,oBAAoB,QAAQ;AACxC,cAAM,SAAS,OAAO,oBAAoB,CAAC;AAC3C,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,eAAO,oBAAoB,MAAM;AACjC,aAAK,iBAAiB;AAAA,MACxB;AAEA,aAAO,oBAAoB,QAAQ,MAAM,OAAO,oBAAoB,IAAI,CAAC;AACzE,UAAI,UAAU;AACZ,eAAO,yBAAyB;AAAA,MAClC;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,uBACE,UACA,QACiB;AACjB,QAAI,CAAC,KAAK,cAAc;AACtB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAEA,QAAI,kBAAkB,sBAAW;AAC/B,eAAS,0BAA0B,UAAU,MAAM;AAAA,IACrD;AAEA,UAAM,WAAW;AACjB,QAAI,EAAE,OAAO,WAAW,WAAW;AAAA,IAEnC;AAEA,UAAM,YAAY,KAAK,MAAM,kBAAkB,MAAM,QAAQ;AAC7D,QAAI,CAAC,WAAW;AACd,YAAM,IAAI,MAAM,+DAA+D;AAAA,IACjF;AAEA,WAAO,KAAK,aAAa,WAAW,UAAU,SAAS;AAAA,EACzD;AAAA,EAEA,MAAM,2BAA2B;AAC/B,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,oBAAoB;AAClE,WAAK,QACF,MAAM,EAAE,UAAU,KAAK,eAAe,GAAG,CAAC,EAC1C,MAAM,yEAAyE;AAClF;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,UAAI,KAAK,MAAM,uBAAuB,CAAC,KAAK,kBAAkB;AAC5D;AAAA,MACF;AACA,WAAK,sBAAsB;AAAA,IAC7B;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAIA,QAAI,KAAK,iBAAiB,MAAM;AAC9B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,YAAI,CAAC,OAAO,QAAS;AACrB,YAAI,OAAO,mBAAoB,QAAO,UAAU;AAAA,MAClD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,UAAU,KAAK,mBAAmB,GAAG,CAAC,EAAE,MAAM,uBAAuB;AAE1F,QAAI,KAAK,iBAAiB;AACxB,YAAM,sBAAsB,KAAK,IAAI,IAAI,KAAK;AAC9C,YAAM,qBAAqB,KAAK;AAAA,SAC7B,KAAK,4BAA4B,KAAK,KAAK;AAAA,QAC5C;AAAA,MACF;AACA,YAAM,UAA8B;AAAA,QAClC,WAAW,KAAK,IAAI;AAAA,QACpB,YAAY,KAAK,mBAAmB;AAAA,QACpC,qBAAqB;AAAA,QACrB;AAAA,MACF;AACA,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C;AAEA,SAAK,qBAAqB,KAAK,kBAAkB;AACjD,SAAK,qBAAqB;AAC1B,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,uBAAuB;AACrB,QACE,CAAC,KAAK,kBACN,CAAC,KAAK,eAAe,sBACrB,KAAK,eAAe,aACpB;AACA;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,sBAAsB,GAAG;AAGtC,YAAM,eAAe,KAAK,MAAM,cAAc,cAAc;AAAA,QAC1D,KAAK;AAAA,MACP;AACA,UAAI,aAAa,SAAS,KAAK,MAAM,mBAAmB;AACtD;AAAA,MACF;AAAA,IACF;AACA,SAAK,eAAe,UAAU;AAAA,EAChC;AAAA,EAEA,qBAAqB,QAAsB;AACzC,SAAK,aAAa,IAAI,MAAM;AAC5B,SAAK,aAAa,IAAI,mBAAmB,cAAc;AACvD,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA;AAAA,EAGA,MAAM,QAAQ;AAv6BhB;AAw6BI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AAEA,eAAK,UAAL,mBAAY,mBAAmB,0BAAU;AAAA,EAE3C;AACF;AAEA,gBAAgB,0BACd,UACA,QACuB;AAp7BzB;AAq7BE,QAAM,YAAY,KAAK,IAAI;AAC3B,MAAI,aAAa;AACjB,mBAAiB,SAAS,QAAQ;AAChC,UAAM,WAAU,WAAM,QAAQ,CAAC,MAAf,mBAAkB,MAAM;AACxC,QAAI,CAAC,QAAS;AAEd,QAAI,YAAY;AACd,mBAAa;AACb,0BAAI,EACD,MAAM,EAAE,UAAU,SAAS,KAAK,MAAM,KAAK,IAAI,IAAI,SAAS,EAAE,CAAC,EAC/D,MAAM,0BAA0B;AAAA,IACrC;AACA,UAAM;AAAA,EACR;AACF;AAGA,MAAM,wBAAwB;AAAA;AAAA,EAEnB,cAAc;AAAA,EACd,4BAA4B;AAAA,EAC5B,4BAA4B;AAAA;AAAA,EAErC;AAAA,EACA;AAAA,EACA,oBAAoB,IAAI,oBAAO;AAAA,EAC/B,uBAAuB;AAAA,EACvB,2BAA2B;AAAA,EAC3B,YAAY;AAAA,EACZ;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,cAAmC,qBAA6B;AAC1E,SAAK,gBAAgB;AACrB,SAAK,oBAAoB;AACzB,SAAK,wBAAwB;AAAA,EAC/B;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,CAAC,KAAK,kBAAkB;AAAA,EACjC;AAAA,EAEA,uBAAuB,YAAoB;AACzC,SAAK,uBAAuB,WAAW,KAAK;AAC5C,QAAI,KAAK,UAAW;AAEpB,UAAM,uBACJ,KAAK,IAAI,IAAI,KAAK,2BAA2B,KAAK;AACpD,QAAI,QAAQ,uBAAuB,KAAK,oBAAoB,KAAK;AACjE,YAAQ,KAAK,oBAAoB,IAAI,QAAQ,KAAK,4BAA4B;AAE9E,SAAK,KAAK,KAAK;AAAA,EACjB;AAAA;AAAA,EAGA,qBAAqB,GAAa;AA7+BpC;AA8+BI,SAAK,YAAY;AACjB,QAAI,KAAK,YAAY;AACnB,iBAAK,WAAL,mBAAa;AAAA,IACf;AAAA,EACF;AAAA;AAAA,EAGA,mBAAmB,GAAa;AAC9B,SAAK,YAAY;AACjB,SAAK,2BAA2B,KAAK,IAAI;AAEzC,QAAI,KAAK,sBAAsB;AAC7B,YAAM,QAAQ,KAAK,oBAAoB,IACnC,KAAK,oBAAoB,KAAK,4BAC9B;AACJ,WAAK,KAAK,KAAK;AAAA,IACjB;AAAA,EACF;AAAA;AAAA,EAIA,sBAA+B;AAC7B,WACE,KAAK,qBAAqB,SAAS,KACnC,KAAK,YAAY,SAAS,KAAK,qBAAqB,KAAK,qBAAqB,SAAS,CAAC,CAAE;AAAA,EAE9F;AAAA,EAEA,eAAe;AACb,SAAK,uBAAuB;AAC5B,SAAK,2BAA2B;AAAA,EAClC;AAAA,EAEA,KAAK,OAAe;AA/gCtB;AAghCI,UAAM,UAAU,OAAOH,QAAe,WAAwB;AAC5D,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,aAAa;AAClB,aAAK,cAAc;AAAA,MACrB,GAAGA,MAAK;AACR,aAAO,iBAAiB,SAAS,MAAM;AACrC,qBAAa,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAEA,eAAK,WAAL,mBAAa;AACb,SAAK,SAAS,IAAI,gBAAgB;AAClC,SAAK,oBAAoB,IAAI,oBAAO;AACpC,SAAK,qBAAqB,QAAQ,OAAO,KAAK,OAAO,MAAM;AAAA,EAC7D;AACF;","names":["import_llm","VPAEvent","BasicSentenceTokenizer","BasicWordTokenizer","EventEmitter","STTStreamAdapter","TTSStreamAdapter","participant","delay","resolve","isUsingTools","task"]}
+{"version":3,"sources":["../../src/pipeline/pipeline_agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { LocalTrackPublication, RemoteParticipant, Room } from '@livekit/rtc-node';\nimport {\n  AudioSource,\n  LocalAudioTrack,\n  RoomEvent,\n  TrackPublishOptions,\n  TrackSource,\n} from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { randomUUID } from 'node:crypto';\nimport EventEmitter from 'node:events';\nimport type {\n  CallableFunctionResult,\n  FunctionCallInfo,\n  FunctionContext,\n  LLM,\n} from '../llm/index.js';\nimport { LLMEvent, LLMStream } from '../llm/index.js';\nimport { ChatContext, ChatMessage, ChatRole } from '../llm/index.js';\nimport { log } from '../log.js';\nimport type { AgentMetrics, PipelineEOUMetrics } from '../metrics/base.js';\nimport { type STT, StreamAdapter as STTStreamAdapter, SpeechEventType } from '../stt/index.js';\nimport {\n  SentenceTokenizer as BasicSentenceTokenizer,\n  WordTokenizer as BasicWordTokenizer,\n  hyphenateWord,\n} from '../tokenize/basic/index.js';\nimport type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';\nimport { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';\nimport type { TTS } from '../tts/index.js';\nimport { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { SpeechSource, SynthesisHandle } from './agent_output.js';\nimport { AgentOutput } from './agent_output.js';\nimport { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';\nimport { HumanInput, HumanInputEvent } from './human_input.js';\nimport { SpeechHandle } from './speech_handle.js';\n\nexport type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';\nexport const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';\nlet speechData: { sequenceId: string } | undefined;\n\nexport type BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n) => LLMStream | false | void | Promise<LLMStream | false | void>;\n\nexport type BeforeTTSCallback = (\n  agent: VoicePipelineAgent,\n  source: string | AsyncIterable<string>,\n) => SpeechSource;\n\nexport enum VPAEvent {\n  USER_STARTED_SPEAKING,\n  USER_STOPPED_SPEAKING,\n  AGENT_STARTED_SPEAKING,\n  AGENT_STOPPED_SPEAKING,\n  USER_SPEECH_COMMITTED,\n  AGENT_SPEECH_COMMITTED,\n  AGENT_SPEECH_INTERRUPTED,\n  FUNCTION_CALLS_COLLECTED,\n  FUNCTION_CALLS_FINISHED,\n  METRICS_COLLECTED,\n}\n\nexport type VPACallbacks = {\n  [VPAEvent.USER_STARTED_SPEAKING]: () => void;\n  [VPAEvent.USER_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STARTED_SPEAKING]: () => void;\n  [VPAEvent.AGENT_STOPPED_SPEAKING]: () => void;\n  [VPAEvent.USER_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_COMMITTED]: (msg: ChatMessage) => void;\n  [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;\n  [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;\n  [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;\n  [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;\n};\n\ninterface TurnDetector {\n  unlikelyThreshold: number;\n  supportsLanguage: (language?: string) => boolean;\n  predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;\n}\n\nexport class AgentCallContext {\n  #agent: VoicePipelineAgent;\n  #llmStream: LLMStream;\n  #metadata = new Map<string, any>();\n  #extraChatMessages: ChatMessage[] = [];\n  static #current: AgentCallContext;\n\n  constructor(agent: VoicePipelineAgent, llmStream: LLMStream) {\n    this.#agent = agent;\n    this.#llmStream = llmStream;\n    AgentCallContext.#current = this;\n  }\n\n  static getCurrent(): AgentCallContext {\n    return AgentCallContext.#current;\n  }\n\n  get agent(): VoicePipelineAgent {\n    return this.#agent;\n  }\n\n  storeMetadata(key: string, value: any) {\n    this.#metadata.set(key, value);\n  }\n\n  getMetadata(key: string, orDefault: any = undefined) {\n    return this.#metadata.get(key) || orDefault;\n  }\n\n  get llmStream(): LLMStream {\n    return this.#llmStream;\n  }\n\n  get extraChatMessages() {\n    return this.#extraChatMessages;\n  }\n\n  addExtraChatMessage(message: ChatMessage) {\n    this.#extraChatMessages.push(message);\n  }\n}\n\nconst defaultBeforeLLMCallback: BeforeLLMCallback = (\n  agent: VoicePipelineAgent,\n  chatCtx: ChatContext,\n): LLMStream => {\n  return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });\n};\n\nconst defaultBeforeTTSCallback: BeforeTTSCallback = (\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  _: VoicePipelineAgent,\n  text: string | AsyncIterable<string>,\n): string | AsyncIterable<string> => {\n  return text;\n};\n\nexport interface AgentTranscriptionOptions {\n  /** Whether to forward the user transcription to the client */\n  userTranscription: boolean;\n  /** Whether to forward the agent transcription to the client */\n  agentTranscription: boolean;\n  /**\n   * The speed at which the agent's speech transcription is forwarded to the client.\n   * We try to mimic the agent's speech speed by adjusting the transcription speed.\n   */\n  agentTranscriptionSpeech: number;\n  /**\n   * The tokenizer used to split the speech into sentences.\n   * This is used to decide when to mark a transcript as final for the agent transcription.\n   */\n  sentenceTokenizer: SentenceTokenizer;\n  /**\n   * The tokenizer used to split the speech into words.\n   * This is used to simulate the \"interim results\" of the agent transcription.\n   */\n  wordTokenizer: WordTokenizer;\n  /**\n   * A function that takes a string (word) as input and returns a list of strings,\n   * representing the hyphenated parts of the word.\n   */\n  hyphenateWord: (word: string) => string[];\n}\n\nconst defaultAgentTranscriptionOptions: AgentTranscriptionOptions = {\n  userTranscription: true,\n  agentTranscription: true,\n  agentTranscriptionSpeech: 1,\n  sentenceTokenizer: new BasicSentenceTokenizer(),\n  wordTokenizer: new BasicWordTokenizer(false),\n  hyphenateWord: hyphenateWord,\n};\n\nexport interface VPAOptions {\n  /** Chat context for the assistant. */\n  chatCtx?: ChatContext;\n  /** Function context for the assistant. */\n  fncCtx?: FunctionContext;\n  /** Whether to allow the user to interrupt the assistant. */\n  allowInterruptions: boolean;\n  /** Minimum duration of speech to consider for interruption. */\n  interruptSpeechDuration: number;\n  /** Minimum number of words to consider for interuption. This may increase latency. */\n  interruptMinWords: number;\n  /** Delay to wait before considering the user speech done. */\n  minEndpointingDelay: number;\n  maxNestedFncCalls: number;\n  /* Whether to preemptively synthesize responses. */\n  preemptiveSynthesis: boolean;\n  /*\n   * Callback called when the assistant is about to synthesize a reply.\n   *\n   * @remarks\n   * Returning void will create a default LLM stream.\n   * You can also return your own LLM stream by calling `llm.chat()`.\n   * Returning `false` ill cancel the synthesis of the reply.\n   */\n  beforeLLMCallback: BeforeLLMCallback;\n  /*\n   * Callback called when the assistant is about to synthesize speech.\n   *\n   * @remarks\n   * This can be used to customize text before synthesis\n   * (e.g. editing the pronunciation of a word).\n   */\n  beforeTTSCallback: BeforeTTSCallback;\n  /** Options for assistant transcription. */\n  transcription: AgentTranscriptionOptions;\n  /** Turn detection model to use. */\n  turnDetector?: TurnDetector;\n}\n\nconst defaultVPAOptions: VPAOptions = {\n  chatCtx: new ChatContext(),\n  allowInterruptions: true,\n  interruptSpeechDuration: 50,\n  interruptMinWords: 0,\n  minEndpointingDelay: 500,\n  maxNestedFncCalls: 1,\n  preemptiveSynthesis: false,\n  beforeLLMCallback: defaultBeforeLLMCallback,\n  beforeTTSCallback: defaultBeforeTTSCallback,\n  transcription: defaultAgentTranscriptionOptions,\n};\n\n/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */\nexport class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<VPACallbacks>) {\n  /** Minimum time played for the user speech to be committed to the chat context. */\n  readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n\n  #vad: VAD;\n  #stt: STT;\n  #llm: LLM;\n  #tts: TTS;\n  #opts: VPAOptions;\n  #humanInput?: HumanInput;\n  #agentOutput?: AgentOutput;\n  #trackPublishedFut = new Future();\n  #pendingAgentReply?: SpeechHandle;\n  #agentReplyTask?: CancellablePromise<void>;\n  #playingSpeech?: SpeechHandle;\n  transcribedText = '';\n  #transcribedInterimText = '';\n  #speechQueueOpen = new Future();\n  #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();\n  #updateStateTask?: CancellablePromise<void>;\n  #started = false;\n  #room?: Room;\n  #participant: RemoteParticipant | string | null = null;\n  #deferredValidation: DeferredReplyValidation;\n  #logger = log();\n  #agentPublication?: LocalTrackPublication;\n  #lastFinalTranscriptTime?: number;\n  #lastSpeechTime?: number;\n  #transcriptionId?: string;\n  #agentTranscribedText = '';\n\n  constructor(\n    /** Voice Activity Detection instance. */\n    vad: VAD,\n    /** Speech-to-Text instance. */\n    stt: STT,\n    /** Large Language Model instance. */\n    llm: LLM,\n    /** Text-to-Speech instance. */\n    tts: TTS,\n    /** Additional VoicePipelineAgent options. */\n    opts: Partial<VPAOptions> = defaultVPAOptions,\n  ) {\n    super();\n\n    this.#opts = { ...defaultVPAOptions, ...opts };\n\n    if (!stt.capabilities.streaming) {\n      stt = new STTStreamAdapter(stt, vad);\n    }\n\n    if (!tts.capabilities.streaming) {\n      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());\n    }\n\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#llm = llm;\n    this.#tts = tts;\n\n    this.#deferredValidation = new DeferredReplyValidation(\n      this.#validateReplyIfPossible.bind(this),\n      this.#opts.minEndpointingDelay,\n      this,\n      this.#opts.turnDetector,\n    );\n  }\n\n  get fncCtx(): FunctionContext | undefined {\n    return this.#opts.fncCtx;\n  }\n\n  set fncCtx(ctx: FunctionContext) {\n    this.#opts.fncCtx = ctx;\n  }\n\n  get chatCtx(): ChatContext {\n    return this.#opts.chatCtx!;\n  }\n\n  get llm(): LLM {\n    return this.#llm;\n  }\n\n  get tts(): TTS {\n    return this.#tts;\n  }\n\n  get stt(): STT {\n    return this.#stt;\n  }\n\n  get vad(): VAD {\n    return this.#vad;\n  }\n\n  /** Start the voice assistant. */\n  start(\n    /** The room to connect to. */\n    room: Room,\n    /**\n     * The participant to listen to.\n     *\n     * @remarks\n     * Can be a participant or an identity.\n     * If omitted, the first participant in the room will be selected.\n     */\n    participant: RemoteParticipant | string | null = null,\n  ) {\n    if (this.#started) {\n      throw new Error('voice assistant already started');\n    }\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {\n      if (!speechData) return;\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });\n    });\n\n    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {\n      if (!speechData) return;\n      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });\n    });\n\n    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    });\n\n    room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {\n      // automatically link to the first participant that connects, if not already linked\n      if (this.#participant) {\n        return;\n      }\n      this.#linkParticipant.call(this, participant.identity!);\n    });\n\n    this.#room = room;\n    this.#participant = participant;\n\n    if (participant) {\n      if (typeof participant === 'string') {\n        this.#linkParticipant(participant);\n      } else {\n        this.#linkParticipant(participant.identity!);\n      }\n    }\n\n    this.#run();\n  }\n\n  /** Play a speech source through the voice assistant. */\n  async say(\n    source: string | LLMStream | AsyncIterable<string>,\n    allowInterruptions = true,\n    addToChatCtx = true,\n  ): Promise<SpeechHandle> {\n    await this.#trackPublishedFut.await;\n\n    let callContext: AgentCallContext | undefined;\n    let fncSource: string | AsyncIterable<string> | undefined;\n    if (addToChatCtx) {\n      callContext = AgentCallContext.getCurrent();\n      if (source instanceof LLMStream) {\n        this.#logger.warn('LLMStream will be ignored for function call chat context');\n      } else if (typeof source === 'string') {\n        fncSource = source;\n      } else {\n        fncSource = source;\n        source = new AsyncIterableQueue<string>();\n      }\n    }\n\n    const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);\n    const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);\n    newHandle.initialize(source, synthesisHandle);\n\n    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {\n      this.#playingSpeech.addNestedSpeech(newHandle);\n    } else {\n      this.#addSpeechForPlayout(newHandle);\n    }\n\n    if (callContext && fncSource) {\n      let text: string;\n      if (typeof source === 'string') {\n        text = fncSource as string;\n      } else {\n        text = '';\n        for await (const chunk of fncSource) {\n          (source as AsyncIterableQueue<string>).put(chunk);\n          text += chunk;\n        }\n        (source as AsyncIterableQueue<string>).close();\n      }\n\n      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));\n      this.#logger.child({ text }).debug('added speech to function call chat context');\n    }\n\n    return newHandle;\n  }\n\n  #updateState(state: AgentState, delay = 0) {\n    const runTask = (delay: number): CancellablePromise<void> => {\n      return new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n        await new Promise((resolve) => setTimeout(resolve, delay));\n        if (this.#room?.isConnected) {\n          if (!cancelled) {\n            await this.#room.localParticipant?.setAttributes({ [AGENT_STATE_ATTRIBUTE]: state });\n          }\n        }\n        resolve();\n      });\n    };\n\n    if (this.#updateStateTask) {\n      this.#updateStateTask.cancel();\n    }\n\n    this.#updateStateTask = runTask(delay);\n  }\n\n  #linkParticipant(participantIdentity: string): void {\n    if (!this.#room) {\n      this.#logger.error('Room is not set');\n      return;\n    }\n\n    this.#participant = this.#room.remoteParticipants.get(participantIdentity) || null;\n    if (!this.#participant) {\n      this.#logger.error(`Participant with identity ${participantIdentity} not found`);\n      return;\n    }\n\n    this.#humanInput = new HumanInput(this.#room, this.#vad, this.#stt, this.#participant);\n    this.#humanInput.on(HumanInputEvent.START_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STARTED_SPEAKING);\n      this.#deferredValidation.onHumanStartOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.VAD_INFERENCE_DONE, (event) => {\n      if (!this.#trackPublishedFut.done) {\n        return;\n      }\n      if (!this.#agentOutput) {\n        throw new Error('agent output is undefined');\n      }\n\n      let tv = 1;\n      if (this.#opts.allowInterruptions) {\n        tv = Math.max(0, 1 - event.probability);\n        this.#agentOutput.playout.targetVolume = tv;\n      }\n\n      if (event.speechDuration >= this.#opts.interruptSpeechDuration) {\n        this.#interruptIfPossible();\n      }\n\n      if (event.rawAccumulatedSpeech > 0) {\n        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;\n      }\n    });\n    this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {\n      this.emit(VPAEvent.USER_STOPPED_SPEAKING);\n      this.#deferredValidation.onHumanEndOfSpeech(event);\n    });\n    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n      this.#transcribedInterimText = event.alternatives![0].text;\n\n      this.#room!.localParticipant!.publishTranscription({\n        participantIdentity: this.#humanInput!.participant.identity,\n        trackSid: this.#humanInput!.subscribedTrack!.sid!,\n        segments: [\n          {\n            text: this.#transcribedInterimText,\n            id: this.#transcriptionId,\n            final: true,\n            startTime: BigInt(0),\n            endTime: BigInt(0),\n            language: '',\n          },\n        ],\n      });\n    });\n    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {\n      const newTranscript = event.alternatives![0].text;\n      if (!newTranscript) return;\n\n      if (!this.#transcriptionId) {\n        this.#transcriptionId = randomUUID();\n      }\n\n      this.#lastFinalTranscriptTime = Date.now();\n      this.transcribedText += (this.transcribedText ? ' ' : '') + newTranscript;\n\n      this.#room!.localParticipant!.publishTranscription({\n        participantIdentity: this.#humanInput!.participant.identity,\n        trackSid: this.#humanInput!.subscribedTrack!.sid!,\n        segments: [\n          {\n            text: this.transcribedText,\n            id: this.#transcriptionId,\n            final: true,\n            startTime: BigInt(0),\n            endTime: BigInt(0),\n            language: '',\n          },\n        ],\n      });\n      this.#transcriptionId = undefined;\n\n      if (\n        this.#opts.preemptiveSynthesis &&\n        (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)\n      ) {\n        this.#synthesizeAgentReply();\n      }\n\n      this.#deferredValidation.onHumanFinalTranscript(newTranscript);\n\n      const words = this.#opts.transcription.wordTokenizer.tokenize(newTranscript);\n      if (words.length >= 3) {\n        // VAD can sometimes not detect that the human is speaking.\n        // to make the interruption more reliable, we also interrupt on the final transcript.\n        this.#interruptIfPossible();\n      }\n    });\n  }\n\n  async #run() {\n    this.#updateState('initializing');\n    const audioSource = new AudioSource(this.#tts.sampleRate, this.#tts.numChannels);\n    const track = LocalAudioTrack.createAudioTrack('assistant_voice', audioSource);\n    this.#agentPublication = await this.#room?.localParticipant?.publishTrack(\n      track,\n      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n    );\n\n    const agentPlayout = new AgentPlayout(audioSource);\n    this.#agentOutput = new AgentOutput(agentPlayout, this.#tts);\n\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STARTED, () => {\n      this.emit(VPAEvent.AGENT_STARTED_SPEAKING);\n      this.#updateState('speaking');\n    });\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    agentPlayout.on(AgentPlayoutEvent.PLAYOUT_STOPPED, (_) => {\n      this.emit(VPAEvent.AGENT_STOPPED_SPEAKING);\n      this.#updateState('listening');\n    });\n\n    this.#trackPublishedFut.resolve();\n\n    while (true) {\n      await this.#speechQueueOpen.await;\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        this.#playingSpeech = undefined;\n      }\n      this.#speechQueueOpen = new Future();\n    }\n  }\n\n  #synthesizeAgentReply() {\n    this.#pendingAgentReply?.cancel();\n    if (this.#humanInput && this.#humanInput.speaking) {\n      this.#updateState('thinking', 200);\n    }\n\n    this.#pendingAgentReply = SpeechHandle.createAssistantReply(\n      this.#opts.allowInterruptions,\n      true,\n      this.transcribedText,\n    );\n    const newHandle = this.#pendingAgentReply;\n    this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);\n  }\n\n  #synthesizeAnswerTask(\n    oldTask: CancellablePromise<void> | undefined,\n    handle?: SpeechHandle,\n  ): CancellablePromise<void> {\n    return new CancellablePromise(async (resolve, _, onCancel) => {\n      let cancelled = false;\n      onCancel(() => {\n        cancelled = true;\n      });\n\n      if (oldTask) {\n        await gracefullyCancel(oldTask);\n      }\n\n      const copiedCtx = this.chatCtx.copy();\n      const playingSpeech = this.#playingSpeech;\n      if (playingSpeech && playingSpeech.initialized) {\n        if (\n          (!playingSpeech.userQuestion || playingSpeech.userCommitted) &&\n          !playingSpeech.speechCommitted\n        ) {\n          // the speech is playing but not committed yet,\n          // add it to the chat context for this new reply synthesis\n          copiedCtx.messages.push(\n            ChatMessage.create({\n              text: playingSpeech.synthesisHandle.text,\n              role: ChatRole.ASSISTANT,\n            }),\n          );\n        }\n      }\n\n      copiedCtx.messages.push(\n        ChatMessage.create({\n          text: handle?.userQuestion,\n          role: ChatRole.USER,\n        }),\n      );\n\n      speechData = { sequenceId: handle!.id };\n\n      try {\n        if (cancelled) resolve();\n        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);\n        if (llmStream === false) {\n          handle?.cancel();\n          return;\n        }\n\n        if (cancelled) resolve();\n        // fallback to default impl if no custom/user stream is returned\n        if (!(llmStream instanceof LLMStream)) {\n          llmStream = (await defaultBeforeLLMCallback(this, copiedCtx)) as LLMStream;\n        }\n\n        if (handle!.interrupted) {\n          return;\n        }\n\n        const synthesisHandle = this.#synthesizeAgentSpeech(handle!.id, llmStream);\n        handle!.initialize(llmStream, synthesisHandle);\n      } finally {\n        speechData = undefined;\n      }\n      resolve();\n    });\n  }\n\n  async #playSpeech(handle: SpeechHandle) {\n    try {\n      await handle.waitForInitialization();\n    } catch {\n      return;\n    }\n    await this.#agentPublication!.waitForSubscription();\n    const synthesisHandle = handle.synthesisHandle;\n    if (synthesisHandle.interrupted) return;\n\n    const userQuestion = handle.userQuestion;\n    const playHandle = synthesisHandle.play();\n    const joinFut = playHandle.join();\n\n    const commitUserQuestionIfNeeded = () => {\n      if (!userQuestion || synthesisHandle.interrupted || handle.userCommitted) return;\n      const isUsingTools =\n        handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n\n      // make sure at least some speech was played before committing the user message\n      // since we try to validate as fast as possible it is possible the agent gets interrupted\n      // really quickly (barely audible), we don't want to mark this question as \"answered\".\n      if (\n        handle.allowInterruptions &&\n        !isUsingTools &&\n        playHandle.timePlayed < this.MIN_TIME_PLAYED_FOR_COMMIT &&\n        !joinFut.done\n      ) {\n        return;\n      }\n\n      this.#logger.child({ userTranscript: userQuestion }).debug('committed user transcript');\n      const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });\n      this.chatCtx.messages.push(userMsg);\n      this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);\n\n      this.transcribedText = this.transcribedText.slice(userQuestion.length);\n      handle.markUserCommitted();\n    };\n\n    // wait for the playHandle to finish and check every 1s if user question should be committed\n    commitUserQuestionIfNeeded();\n\n    while (!joinFut.done) {\n      await new Promise<void>(async (resolve) => {\n        setTimeout(resolve, 500);\n        await joinFut.await;\n        resolve();\n      });\n      commitUserQuestionIfNeeded();\n      if (handle.interrupted) break;\n    }\n    commitUserQuestionIfNeeded();\n\n    let collectedText = this.#agentTranscribedText;\n    const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;\n    const interrupted = handle.interrupted;\n\n    if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {\n      if (handle.extraToolsMessages) {\n        this.chatCtx.messages.push(...handle.extraToolsMessages);\n      }\n      if (interrupted) {\n        collectedText += '…';\n      }\n\n      const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });\n      this.chatCtx.messages.push(msg);\n\n      handle.markSpeechCommitted();\n      if (interrupted) {\n        this.emit(VPAEvent.AGENT_SPEECH_INTERRUPTED, msg);\n      } else {\n        this.emit(VPAEvent.AGENT_SPEECH_COMMITTED, msg);\n      }\n\n      this.#logger\n        .child({\n          agentTranscript: collectedText,\n          interrupted,\n          speechId: handle.id,\n        })\n        .debug('committed agent speech');\n\n      handle.setDone();\n    }\n\n    const executeFunctionCalls = async () => {\n      // if the answer is using tools, execute the functions and automatically generate\n      // a response to the user question from the returned values\n      if (!isUsingTools || interrupted) return;\n\n      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {\n        this.#logger\n          .child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth })\n          .warn('max function calls nested depth reached');\n        return;\n      }\n\n      if (userQuestion && !handle.userCommitted) {\n        throw new Error('user speech should have been committed before using tools');\n      }\n      const llmStream = handle.source;\n      const newFunctionCalls = llmStream.functionCalls;\n\n      new AgentCallContext(this, llmStream);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);\n      const calledFuncs: FunctionCallInfo[] = [];\n      for (const func of newFunctionCalls) {\n        const task = func.func.execute(func.params).then(\n          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),\n          (error) => ({ name: func.name, toolCallId: func.toolCallId, error }),\n        );\n        calledFuncs.push({ ...func, task });\n        this.#logger\n          .child({ function: func.name, speechId: handle.id })\n          .debug('executing AI function');\n        try {\n          await task;\n        } catch {\n          this.#logger\n            .child({ function: func.name, speechId: handle.id })\n            .error('error executing AI function');\n        }\n      }\n\n      const toolCallsInfo = [];\n      const toolCallsResults = [];\n      for (const fnc of calledFuncs) {\n        // ignore the function calls that return void\n        const task = await fnc.task;\n        if (!task || task.result === undefined) continue;\n        toolCallsInfo.push(fnc);\n        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));\n      }\n\n      if (!toolCallsInfo.length) return;\n\n      // generate an answer from the tool calls\n      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];\n      extraToolsMessages.push(...toolCallsResults);\n\n      // create a nested speech handle\n      const newSpeechHandle = SpeechHandle.createToolSpeech(\n        handle.allowInterruptions,\n        handle.addToChatCtx,\n        handle.fncNestedDepth + 1,\n        extraToolsMessages,\n      );\n\n      // synthesize the tool speech with the chat ctx from llmStream\n      const chatCtx = handle.source.chatCtx.copy();\n      chatCtx.messages.push(...extraToolsMessages);\n      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);\n\n      const answerLLMStream = this.llm.chat({\n        chatCtx,\n        fncCtx: this.fncCtx,\n      });\n\n      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);\n      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);\n      handle.addNestedSpeech(newSpeechHandle);\n\n      this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);\n    };\n\n    let finished = false;\n    const task = executeFunctionCalls().then(() => {\n      finished = true;\n    });\n    while (!handle.nestedSpeechFinished) {\n      const changed = handle.nestedSpeechChanged();\n      await Promise.race([changed, task]);\n      while (handle.nestedSpeechHandles.length) {\n        const speech = handle.nestedSpeechHandles[0]!;\n        this.#playingSpeech = speech;\n        await this.#playSpeech(speech);\n        handle.nestedSpeechHandles.shift();\n        this.#playingSpeech = handle;\n      }\n\n      handle.nestedSpeechHandles.forEach(() => handle.nestedSpeechHandles.pop());\n      if (finished) {\n        handle.markNestedSpeechFinished();\n      }\n    }\n    handle.setDone();\n  }\n\n  #synthesizeAgentSpeech(\n    speechId: string,\n    source: string | LLMStream | AsyncIterable<string>,\n  ): SynthesisHandle {\n    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);\n    synchronizer.on('textUpdated', (text) => {\n      this.#agentTranscribedText = text.text;\n      this.#room!.localParticipant!.publishTranscription({\n        participantIdentity: this.#room!.localParticipant!.identity,\n        trackSid: this.#agentPublication!.sid!,\n        segments: [text],\n      });\n    });\n\n    if (!this.#agentOutput) {\n      throw new Error('agent output should be initialized when ready');\n    }\n\n    if (source instanceof LLMStream) {\n      source = llmStreamToStringIterable(speechId, source);\n    }\n\n    const ogSource = source;\n    if (!(typeof source === 'string')) {\n      // TODO(nbsp): itertools.tee\n    }\n\n    const ttsSource = this.#opts.beforeTTSCallback(this, ogSource);\n    if (!ttsSource) {\n      throw new Error('beforeTTSCallback must return string or AsyncIterable<string>');\n    }\n\n    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);\n  }\n\n  async #validateReplyIfPossible() {\n    if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {\n      this.#logger\n        .child({ speechId: this.#playingSpeech.id })\n        .debug('skipping validation, agent is speaking and does not allow interruptions');\n      return;\n    }\n\n    if (!this.#pendingAgentReply) {\n      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {\n        return;\n      }\n      this.#synthesizeAgentReply();\n    }\n\n    if (!this.#pendingAgentReply) {\n      throw new Error('pending agent reply is undefined');\n    }\n\n    // in some bad timimg, we could end up with two pushed agent replies inside the speech queue.\n    // so make sure we directly interrupt every reply when validating a new one\n    if (this.#speechQueueOpen.done) {\n      for await (const speech of this.#speechQueue) {\n        if (speech === VoicePipelineAgent.FLUSH_SENTINEL) break;\n        if (!speech.isReply) continue;\n        if (speech.allowInterruptions) speech.interrupt();\n      }\n    }\n\n    this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');\n\n    if (this.#lastSpeechTime) {\n      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;\n      const transcriptionDelay = Math.max(\n        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,\n        0,\n      );\n      const metrics: PipelineEOUMetrics = {\n        timestamp: Date.now(),\n        sequenceId: this.#pendingAgentReply.id,\n        endOfUtteranceDelay: timeSinceLastSpeech,\n        transcriptionDelay,\n      };\n      this.emit(VPAEvent.METRICS_COLLECTED, metrics);\n    }\n\n    this.#addSpeechForPlayout(this.#pendingAgentReply);\n    this.#pendingAgentReply = undefined;\n    this.#transcribedInterimText = '';\n  }\n\n  #interruptIfPossible() {\n    if (\n      !this.#playingSpeech ||\n      !this.#playingSpeech.allowInterruptions ||\n      this.#playingSpeech.interrupted\n    ) {\n      return;\n    }\n\n    if (this.#opts.interruptMinWords !== 0) {\n      // check the final/interim transcribed text for the minimum word count\n      // to interrupt the agent speech\n      const interimWords = this.#opts.transcription.wordTokenizer.tokenize(\n        this.#transcribedInterimText,\n      );\n      if (interimWords.length < this.#opts.interruptMinWords) {\n        return;\n      }\n    }\n    this.#playingSpeech.interrupt();\n  }\n\n  #addSpeechForPlayout(handle: SpeechHandle) {\n    this.#speechQueue.put(handle);\n    this.#speechQueue.put(VoicePipelineAgent.FLUSH_SENTINEL);\n    this.#speechQueueOpen.resolve();\n  }\n\n  /** Close the voice assistant. */\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n\n    this.#room?.removeAllListeners(RoomEvent.ParticipantConnected);\n    // TODO(nbsp): await this.#deferredValidation.close()\n  }\n}\n\nasync function* llmStreamToStringIterable(\n  speechId: string,\n  stream: LLMStream,\n): AsyncIterable<string> {\n  const startTime = Date.now();\n  let firstFrame = true;\n  for await (const chunk of stream) {\n    const content = chunk.choices[0]?.delta.content;\n    if (!content) continue;\n\n    if (firstFrame) {\n      firstFrame = false;\n      log()\n        .child({ speechId, elapsed: Math.round(Date.now() - startTime) })\n        .debug('received first LLM token');\n    }\n    yield content;\n  }\n}\n\n/** This class is used to try to find the best time to validate the agent reply. */\nclass DeferredReplyValidation {\n  // if the STT gives us punctuation, we can try to validate the reply faster.\n  readonly PUNCTUATION = '.!?';\n  readonly PUNCTUATION_REDUCE_FACTOR = 0.75;\n  readonly LATE_TRANSCRIPT_TOLERANCE = 1.5; // late compared to end of speech\n  readonly UNLIKELY_ENDPOINT_DELAY = 6000;\n\n  #validateFunc: () => Promise<void>;\n  #validatingPromise?: Promise<void>;\n  #validatingFuture = new Future();\n  #lastFinalTranscript = '';\n  #lastRecvEndOfSpeechTime = 0;\n  #speaking = false;\n  #endOfSpeechDelay: number;\n  #finalTranscriptDelay: number;\n  #turnDetector?: TurnDetector;\n  #agent: VoicePipelineAgent;\n  #abort?: AbortController;\n\n  constructor(\n    validateFunc: () => Promise<void>,\n    minEndpointingDelay: number,\n    agent: VoicePipelineAgent,\n    turnDetector?: TurnDetector,\n  ) {\n    this.#validateFunc = validateFunc;\n    this.#endOfSpeechDelay = minEndpointingDelay;\n    this.#finalTranscriptDelay = minEndpointingDelay;\n    this.#agent = agent;\n    this.#turnDetector = turnDetector;\n  }\n\n  get validating(): boolean {\n    return !this.#validatingFuture.done;\n  }\n\n  onHumanFinalTranscript(transcript: string) {\n    this.#lastFinalTranscript = transcript.trim();\n    if (this.#speaking) return;\n\n    const hasRecentEndOfSpeech =\n      Date.now() - this.#lastRecvEndOfSpeechTime < this.LATE_TRANSCRIPT_TOLERANCE;\n    let delay = hasRecentEndOfSpeech ? this.#endOfSpeechDelay : this.#finalTranscriptDelay;\n    delay = this.#endWithPunctuation() ? delay * this.PUNCTUATION_REDUCE_FACTOR : 1;\n\n    this.#run(delay);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanStartOfSpeech(_: VADEvent) {\n    this.#speaking = true;\n    if (this.validating) {\n      this.#abort?.abort();\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  onHumanEndOfSpeech(_: VADEvent) {\n    this.#speaking = false;\n    this.#lastRecvEndOfSpeechTime = Date.now();\n\n    if (this.#lastFinalTranscript) {\n      const delay = this.#endWithPunctuation()\n        ? this.#endOfSpeechDelay * this.PUNCTUATION_REDUCE_FACTOR\n        : 1_000;\n      this.#run(delay);\n    }\n  }\n\n  // TODO(nbsp): aclose\n\n  #endWithPunctuation(): boolean {\n    return (\n      this.#lastFinalTranscript.length > 0 &&\n      this.PUNCTUATION.includes(this.#lastFinalTranscript[this.#lastFinalTranscript.length - 1]!)\n    );\n  }\n\n  #resetStates() {\n    this.#lastFinalTranscript = '';\n    this.#lastRecvEndOfSpeechTime = 0;\n  }\n\n  #run(delay: number) {\n    const runTask = async (delay: number, chatCtx: ChatContext, signal: AbortSignal) => {\n      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {\n        const startTime = Date.now();\n        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);\n        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;\n        const elapsed = Date.now() - startTime;\n        if (eotProb < unlikelyThreshold) {\n          delay = this.UNLIKELY_ENDPOINT_DELAY;\n        }\n        delay = Math.max(0, delay - elapsed);\n      }\n      const timeout = setTimeout(() => {\n        this.#resetStates();\n        this.#validateFunc();\n      }, delay);\n      signal.addEventListener('abort', () => {\n        clearTimeout(timeout);\n      });\n    };\n\n    this.#abort?.abort();\n    this.#abort = new AbortController();\n    this.#validatingFuture = new Future();\n    const detectCtx = this.#agent.chatCtx.copy();\n    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });\n    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAMO;AAEP,yBAA2B;AAC3B,yBAAyB;AAOzB,iBAAoC;AACpC,IAAAA,cAAmD;AACnD,iBAAoB;AAEpB,iBAA6E;AAC7E,mBAIO;AAEP,2BAA8D;AAE9D,iBAA4D;AAC5D,mBAAiF;AACjF,iBAAsD;AAEtD,0BAA4B;AAC5B,2BAAgD;AAChD,yBAA4C;AAC5C,2BAA6B;AAGtB,MAAM,wBAAwB;AACrC,IAAI;AAYG,IAAK,WAAL,kBAAKC,cAAL;AACL,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AACA,EAAAA,oBAAA;AAVU,SAAAA;AAAA,GAAA;AAgCL,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA,YAAY,oBAAI,IAAiB;AAAA,EACjC,qBAAoC,CAAC;AAAA,EACrC,OAAO;AAAA,EAEP,YAAY,OAA2B,WAAsB;AAC3D,SAAK,SAAS;AACd,SAAK,aAAa;AAClB,qBAAiB,WAAW;AAAA,EAC9B;AAAA,EAEA,OAAO,aAA+B;AACpC,WAAO,iBAAiB;AAAA,EAC1B;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAc,KAAa,OAAY;AACrC,SAAK,UAAU,IAAI,KAAK,KAAK;AAAA,EAC/B;AAAA,EAEA,YAAY,KAAa,YAAiB,QAAW;AACnD,WAAO,KAAK,UAAU,IAAI,GAAG,KAAK;AAAA,EACpC;AAAA,EAEA,IAAI,YAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,oBAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,oBAAoB,SAAsB;AACxC,SAAK,mBAAmB,KAAK,OAAO;AAAA,EACtC;AACF;AAEA,MAAM,2BAA8C,CAClD,OACA,YACc;AACd,SAAO,MAAM,IAAI,KAAK,EAAE,SAAS,QAAQ,MAAM,OAAO,CAAC;AACzD;AAEA,MAAM,2BAA8C,CAElD,GACA,SACmC;AACnC,SAAO;AACT;AA6BA,MAAM,mCAA8D;AAAA,EAClE,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,aAAAC,kBAAuB;AAAA,EAC9C,eAAe,IAAI,aAAAC,cAAmB,KAAK;AAAA,EAC3C,eAAe;AACjB;AAyCA,MAAM,oBAAgC;AAAA,EACpC,SAAS,IAAI,wBAAY;AAAA,EACzB,oBAAoB;AAAA,EACpB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,eAAe;AACjB;AAGO,MAAM,2BAA4B,mBAAAC,QAAsD;AAAA;AAAA,EAEpF,6BAA6B;AAAA,EACtC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAElE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,qBAAqB,IAAI,oBAAO;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA,kBAAkB;AAAA,EAClB,0BAA0B;AAAA,EAC1B,mBAAmB,IAAI,oBAAO;AAAA,EAC9B,eAAe,IAAI,gCAA4E;AAAA,EAC/F;AAAA,EACA,WAAW;AAAA,EACX;AAAA,EACA,eAAkD;AAAA,EAClD;AAAA,EACA,cAAU,gBAAI;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,wBAAwB;AAAA,EAExB,YAEE,KAEA,KAEA,KAEA,KAEA,OAA4B,mBAC5B;AACA,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAE7C,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,WAAAC,cAAiB,KAAK,GAAG;AAAA,IACrC;AAEA,QAAI,CAAC,IAAI,aAAa,WAAW;AAC/B,YAAM,IAAI,WAAAC,cAAiB,KAAK,IAAI,aAAAJ,kBAAuB,CAAC;AAAA,IAC9D;AAEA,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,OAAO;AAEZ,SAAK,sBAAsB,IAAI;AAAA,MAC7B,KAAK,yBAAyB,KAAK,IAAI;AAAA,MACvC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK,MAAM;AAAA,IACb;AAAA,EACF;AAAA,EAEA,IAAI,SAAsC;AACxC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,OAAO,KAAsB;AAC/B,SAAK,MAAM,SAAS;AAAA,EACtB;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAW;AACb,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAEE,MAQA,cAAiD,MACjD;AACA,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,SAAK,KAAK,GAAG,2BAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,KAAK,GAAG,oBAAS,mBAAmB,CAAC,YAAY;AACpD,UAAI,CAAC,WAAY;AACjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,YAAY,WAAW,WAAW,CAAC;AAAA,IACzF,CAAC;AAED,SAAK,KAAK,GAAG,oBAAS,mBAAmB,CAAC,YAAY;AACpD,UAAI,CAAC,WAAY;AACjB,WAAK,KAAK,2BAA4B,EAAE,GAAG,SAAS,YAAY,WAAW,WAAW,CAAC;AAAA,IACzF,CAAC;AAED,SAAK,KAAK,GAAG,wBAAa,mBAAmB,CAAC,YAAY;AACxD,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C,CAAC;AAED,SAAK,GAAG,0BAAU,sBAAsB,CAACK,iBAAmC;AAE1E,UAAI,KAAK,cAAc;AACrB;AAAA,MACF;AACA,WAAK,iBAAiB,KAAK,MAAMA,aAAY,QAAS;AAAA,IACxD,CAAC;AAED,SAAK,QAAQ;AACb,SAAK,eAAe;AAEpB,QAAI,aAAa;AACf,UAAI,OAAO,gBAAgB,UAAU;AACnC,aAAK,iBAAiB,WAAW;AAAA,MACnC,OAAO;AACL,aAAK,iBAAiB,YAAY,QAAS;AAAA,MAC7C;AAAA,IACF;AAEA,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA,EAGA,MAAM,IACJ,QACA,qBAAqB,MACrB,eAAe,MACQ;AACvB,UAAM,KAAK,mBAAmB;AAE9B,QAAI;AACJ,QAAI;AACJ,QAAI,cAAc;AAChB,oBAAc,iBAAiB,WAAW;AAC1C,UAAI,kBAAkB,sBAAW;AAC/B,aAAK,QAAQ,KAAK,0DAA0D;AAAA,MAC9E,WAAW,OAAO,WAAW,UAAU;AACrC,oBAAY;AAAA,MACd,OAAO;AACL,oBAAY;AACZ,iBAAS,IAAI,gCAA2B;AAAA,MAC1C;AAAA,IACF;AAEA,UAAM,YAAY,kCAAa,sBAAsB,oBAAoB,YAAY;AACrF,UAAM,kBAAkB,KAAK,uBAAuB,UAAU,IAAI,MAAM;AACxE,cAAU,WAAW,QAAQ,eAAe;AAE5C,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,sBAAsB;AACpE,WAAK,eAAe,gBAAgB,SAAS;AAAA,IAC/C,OAAO;AACL,WAAK,qBAAqB,SAAS;AAAA,IACrC;AAEA,QAAI,eAAe,WAAW;AAC5B,UAAI;AACJ,UAAI,OAAO,WAAW,UAAU;AAC9B,eAAO;AAAA,MACT,OAAO;AACL,eAAO;AACP,yBAAiB,SAAS,WAAW;AACnC,UAAC,OAAsC,IAAI,KAAK;AAChD,kBAAQ;AAAA,QACV;AACA,QAAC,OAAsC,MAAM;AAAA,MAC/C;AAEA,kBAAY,oBAAoB,wBAAY,OAAO,EAAE,MAAM,MAAM,qBAAS,UAAU,CAAC,CAAC;AACtF,WAAK,QAAQ,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,4CAA4C;AAAA,IACjF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmB,QAAQ,GAAG;AACzC,UAAM,UAAU,CAACC,WAA4C;AAC3D,aAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AA1bpE;AA2bQ,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AACD,cAAM,IAAI,QAAQ,CAACC,aAAY,WAAWA,UAASD,MAAK,CAAC;AACzD,aAAI,UAAK,UAAL,mBAAY,aAAa;AAC3B,cAAI,CAAC,WAAW;AACd,oBAAM,UAAK,MAAM,qBAAX,mBAA6B,cAAc,EAAE,CAAC,qBAAqB,GAAG,MAAM;AAAA,UACpF;AAAA,QACF;AACA,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,kBAAkB;AACzB,WAAK,iBAAiB,OAAO;AAAA,IAC/B;AAEA,SAAK,mBAAmB,QAAQ,KAAK;AAAA,EACvC;AAAA,EAEA,iBAAiB,qBAAmC;AAClD,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,MAAM,iBAAiB;AACpC;AAAA,IACF;AAEA,SAAK,eAAe,KAAK,MAAM,mBAAmB,IAAI,mBAAmB,KAAK;AAC9E,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,6BAA6B,mBAAmB,YAAY;AAC/E;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,8BAAW,KAAK,OAAO,KAAK,MAAM,KAAK,MAAM,KAAK,YAAY;AACrF,SAAK,YAAY,GAAG,mCAAgB,iBAAiB,CAAC,UAAU;AAC9D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,qBAAqB,KAAK;AAAA,IACrD,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,oBAAoB,CAAC,UAAU;AACjE,UAAI,CAAC,KAAK,mBAAmB,MAAM;AACjC;AAAA,MACF;AACA,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,2BAA2B;AAAA,MAC7C;AAEA,UAAI,KAAK;AACT,UAAI,KAAK,MAAM,oBAAoB;AACjC,aAAK,KAAK,IAAI,GAAG,IAAI,MAAM,WAAW;AACtC,aAAK,aAAa,QAAQ,eAAe;AAAA,MAC3C;AAEA,UAAI,MAAM,kBAAkB,KAAK,MAAM,yBAAyB;AAC9D,aAAK,qBAAqB;AAAA,MAC5B;AAEA,UAAI,MAAM,uBAAuB,GAAG;AAClC,aAAK,kBAAkB,KAAK,IAAI,IAAI,MAAM;AAAA,MAC5C;AAAA,IACF,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,eAAe,CAAC,UAAU;AAC5D,WAAK,KAAK,6BAA8B;AACxC,WAAK,oBAAoB,mBAAmB,KAAK;AAAA,IACnD,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,oBAAoB,CAAC,UAAU;AACjE,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,uBAAmB,+BAAW;AAAA,MACrC;AACA,WAAK,0BAA0B,MAAM,aAAc,CAAC,EAAE;AAEtD,WAAK,MAAO,iBAAkB,qBAAqB;AAAA,QACjD,qBAAqB,KAAK,YAAa,YAAY;AAAA,QACnD,UAAU,KAAK,YAAa,gBAAiB;AAAA,QAC7C,UAAU;AAAA,UACR;AAAA,YACE,MAAM,KAAK;AAAA,YACX,IAAI,KAAK;AAAA,YACT,OAAO;AAAA,YACP,WAAW,OAAO,CAAC;AAAA,YACnB,SAAS,OAAO,CAAC;AAAA,YACjB,UAAU;AAAA,UACZ;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AACD,SAAK,YAAY,GAAG,mCAAgB,kBAAkB,CAAC,UAAU;AAC/D,YAAM,gBAAgB,MAAM,aAAc,CAAC,EAAE;AAC7C,UAAI,CAAC,cAAe;AAEpB,UAAI,CAAC,KAAK,kBAAkB;AAC1B,aAAK,uBAAmB,+BAAW;AAAA,MACrC;AAEA,WAAK,2BAA2B,KAAK,IAAI;AACzC,WAAK,oBAAoB,KAAK,kBAAkB,MAAM,MAAM;AAE5D,WAAK,MAAO,iBAAkB,qBAAqB;AAAA,QACjD,qBAAqB,KAAK,YAAa,YAAY;AAAA,QACnD,UAAU,KAAK,YAAa,gBAAiB;AAAA,QAC7C,UAAU;AAAA,UACR;AAAA,YACE,MAAM,KAAK;AAAA,YACX,IAAI,KAAK;AAAA,YACT,OAAO;AAAA,YACP,WAAW,OAAO,CAAC;AAAA,YACnB,SAAS,OAAO,CAAC;AAAA,YACjB,UAAU;AAAA,UACZ;AAAA,QACF;AAAA,MACF,CAAC;AACD,WAAK,mBAAmB;AAExB,UACE,KAAK,MAAM,wBACV,CAAC,KAAK,kBAAkB,KAAK,eAAe,qBAC7C;AACA,aAAK,sBAAsB;AAAA,MAC7B;AAEA,WAAK,oBAAoB,uBAAuB,aAAa;AAE7D,YAAM,QAAQ,KAAK,MAAM,cAAc,cAAc,SAAS,aAAa;AAC3E,UAAI,MAAM,UAAU,GAAG;AAGrB,aAAK,qBAAqB;AAAA,MAC5B;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAO;AA7jBf;AA8jBI,SAAK,aAAa,cAAc;AAChC,UAAM,cAAc,IAAI,4BAAY,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW;AAC/E,UAAM,QAAQ,gCAAgB,iBAAiB,mBAAmB,WAAW;AAC7E,SAAK,oBAAoB,QAAM,gBAAK,UAAL,mBAAY,qBAAZ,mBAA8B;AAAA,MAC3D;AAAA,MACA,IAAI,oCAAoB,EAAE,QAAQ,4BAAY,kBAAkB,CAAC;AAAA;AAGnE,UAAM,eAAe,IAAI,kCAAa,WAAW;AACjD,SAAK,eAAe,IAAI,gCAAY,cAAc,KAAK,IAAI;AAE3D,iBAAa,GAAG,uCAAkB,iBAAiB,MAAM;AACvD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,UAAU;AAAA,IAC9B,CAAC;AAED,iBAAa,GAAG,uCAAkB,iBAAiB,CAAC,MAAM;AACxD,WAAK,KAAK,8BAA+B;AACzC,WAAK,aAAa,WAAW;AAAA,IAC/B,CAAC;AAED,SAAK,mBAAmB,QAAQ;AAEhC,WAAO,MAAM;AACX,YAAM,KAAK,iBAAiB;AAC5B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,aAAK,iBAAiB;AAAA,MACxB;AACA,WAAK,mBAAmB,IAAI,oBAAO;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,wBAAwB;AAjmB1B;AAkmBI,eAAK,uBAAL,mBAAyB;AACzB,QAAI,KAAK,eAAe,KAAK,YAAY,UAAU;AACjD,WAAK,aAAa,YAAY,GAAG;AAAA,IACnC;AAEA,SAAK,qBAAqB,kCAAa;AAAA,MACrC,KAAK,MAAM;AAAA,MACX;AAAA,MACA,KAAK;AAAA,IACP;AACA,UAAM,YAAY,KAAK;AACvB,SAAK,kBAAkB,KAAK,sBAAsB,KAAK,iBAAiB,SAAS;AAAA,EACnF;AAAA,EAEA,sBACE,SACA,QAC0B;AAC1B,WAAO,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC5D,UAAI,YAAY;AAChB,eAAS,MAAM;AACb,oBAAY;AAAA,MACd,CAAC;AAED,UAAI,SAAS;AACX,kBAAM,+BAAiB,OAAO;AAAA,MAChC;AAEA,YAAM,YAAY,KAAK,QAAQ,KAAK;AACpC,YAAM,gBAAgB,KAAK;AAC3B,UAAI,iBAAiB,cAAc,aAAa;AAC9C,aACG,CAAC,cAAc,gBAAgB,cAAc,kBAC9C,CAAC,cAAc,iBACf;AAGA,oBAAU,SAAS;AAAA,YACjB,wBAAY,OAAO;AAAA,cACjB,MAAM,cAAc,gBAAgB;AAAA,cACpC,MAAM,qBAAS;AAAA,YACjB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,gBAAU,SAAS;AAAA,QACjB,wBAAY,OAAO;AAAA,UACjB,MAAM,iCAAQ;AAAA,UACd,MAAM,qBAAS;AAAA,QACjB,CAAC;AAAA,MACH;AAEA,mBAAa,EAAE,YAAY,OAAQ,GAAG;AAEtC,UAAI;AACF,YAAI,UAAW,SAAQ;AACvB,YAAI,YAAY,MAAM,KAAK,MAAM,kBAAkB,MAAM,SAAS;AAClE,YAAI,cAAc,OAAO;AACvB,2CAAQ;AACR;AAAA,QACF;AAEA,YAAI,UAAW,SAAQ;AAEvB,YAAI,EAAE,qBAAqB,uBAAY;AACrC,sBAAa,MAAM,yBAAyB,MAAM,SAAS;AAAA,QAC7D;AAEA,YAAI,OAAQ,aAAa;AACvB;AAAA,QACF;AAEA,cAAM,kBAAkB,KAAK,uBAAuB,OAAQ,IAAI,SAAS;AACzE,eAAQ,WAAW,WAAW,eAAe;AAAA,MAC/C,UAAE;AACA,qBAAa;AAAA,MACf;AACA,cAAQ;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAsB;AACtC,QAAI;AACF,YAAM,OAAO,sBAAsB;AAAA,IACrC,QAAQ;AACN;AAAA,IACF;AACA,UAAM,KAAK,kBAAmB,oBAAoB;AAClD,UAAM,kBAAkB,OAAO;AAC/B,QAAI,gBAAgB,YAAa;AAEjC,UAAM,eAAe,OAAO;AAC5B,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,UAAU,WAAW,KAAK;AAEhC,UAAM,6BAA6B,MAAM;AACvC,UAAI,CAAC,gBAAgB,gBAAgB,eAAe,OAAO,cAAe;AAC1E,YAAME,gBACJ,OAAO,kBAAkB,wBAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AAKtE,UACE,OAAO,sBACP,CAACA,iBACD,WAAW,aAAa,KAAK,8BAC7B,CAAC,QAAQ,MACT;AACA;AAAA,MACF;AAEA,WAAK,QAAQ,MAAM,EAAE,gBAAgB,aAAa,CAAC,EAAE,MAAM,2BAA2B;AACtF,YAAM,UAAU,wBAAY,OAAO,EAAE,MAAM,cAAc,MAAM,qBAAS,KAAK,CAAC;AAC9E,WAAK,QAAQ,SAAS,KAAK,OAAO;AAClC,WAAK,KAAK,+BAAgC,OAAO;AAEjD,WAAK,kBAAkB,KAAK,gBAAgB,MAAM,aAAa,MAAM;AACrE,aAAO,kBAAkB;AAAA,IAC3B;AAGA,+BAA2B;AAE3B,WAAO,CAAC,QAAQ,MAAM;AACpB,YAAM,IAAI,QAAc,OAAO,YAAY;AACzC,mBAAW,SAAS,GAAG;AACvB,cAAM,QAAQ;AACd,gBAAQ;AAAA,MACV,CAAC;AACD,iCAA2B;AAC3B,UAAI,OAAO,YAAa;AAAA,IAC1B;AACA,+BAA2B;AAE3B,QAAI,gBAAgB,KAAK;AACzB,UAAM,eAAe,OAAO,kBAAkB,wBAAa,CAAC,CAAC,OAAO,OAAO,cAAc;AACzF,UAAM,cAAc,OAAO;AAE3B,QAAI,OAAO,iBAAiB,CAAC,gBAAgB,OAAO,gBAAgB;AAClE,UAAI,OAAO,oBAAoB;AAC7B,aAAK,QAAQ,SAAS,KAAK,GAAG,OAAO,kBAAkB;AAAA,MACzD;AACA,UAAI,aAAa;AACf,yBAAiB;AAAA,MACnB;AAEA,YAAM,MAAM,wBAAY,OAAO,EAAE,MAAM,eAAe,MAAM,qBAAS,UAAU,CAAC;AAChF,WAAK,QAAQ,SAAS,KAAK,GAAG;AAE9B,aAAO,oBAAoB;AAC3B,UAAI,aAAa;AACf,aAAK,KAAK,kCAAmC,GAAG;AAAA,MAClD,OAAO;AACL,aAAK,KAAK,gCAAiC,GAAG;AAAA,MAChD;AAEA,WAAK,QACF,MAAM;AAAA,QACL,iBAAiB;AAAA,QACjB;AAAA,QACA,UAAU,OAAO;AAAA,MACnB,CAAC,EACA,MAAM,wBAAwB;AAEjC,aAAO,QAAQ;AAAA,IACjB;AAEA,UAAM,uBAAuB,YAAY;AAGvC,UAAI,CAAC,gBAAgB,YAAa;AAElC,UAAI,OAAO,kBAAkB,KAAK,MAAM,mBAAmB;AACzD,aAAK,QACF,MAAM,EAAE,UAAU,OAAO,IAAI,gBAAgB,OAAO,eAAe,CAAC,EACpE,KAAK,yCAAyC;AACjD;AAAA,MACF;AAEA,UAAI,gBAAgB,CAAC,OAAO,eAAe;AACzC,cAAM,IAAI,MAAM,2DAA2D;AAAA,MAC7E;AACA,YAAM,YAAY,OAAO;AACzB,YAAM,mBAAmB,UAAU;AAEnC,UAAI,iBAAiB,MAAM,SAAS;AAEpC,WAAK,KAAK,kCAAmC,gBAAgB;AAC7D,YAAM,cAAkC,CAAC;AACzC,iBAAW,QAAQ,kBAAkB;AACnC,cAAMC,QAAO,KAAK,KAAK,QAAQ,KAAK,MAAM,EAAE;AAAA,UAC1C,CAAC,YAAY,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,OAAO;AAAA,UACpE,CAAC,WAAW,EAAE,MAAM,KAAK,MAAM,YAAY,KAAK,YAAY,MAAM;AAAA,QACpE;AACA,oBAAY,KAAK,EAAE,GAAG,MAAM,MAAAA,MAAK,CAAC;AAClC,aAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,uBAAuB;AAChC,YAAI;AACF,gBAAMA;AAAA,QACR,QAAQ;AACN,eAAK,QACF,MAAM,EAAE,UAAU,KAAK,MAAM,UAAU,OAAO,GAAG,CAAC,EAClD,MAAM,6BAA6B;AAAA,QACxC;AAAA,MACF;AAEA,YAAM,gBAAgB,CAAC;AACvB,YAAM,mBAAmB,CAAC;AAC1B,iBAAW,OAAO,aAAa;AAE7B,cAAMA,QAAO,MAAM,IAAI;AACvB,YAAI,CAACA,SAAQA,MAAK,WAAW,OAAW;AACxC,sBAAc,KAAK,GAAG;AACtB,yBAAiB,KAAK,wBAAY,6BAA6BA,KAAI,CAAC;AAAA,MACtE;AAEA,UAAI,CAAC,cAAc,OAAQ;AAG3B,YAAM,qBAAqB,CAAC,wBAAY,gBAAgB,eAAe,aAAa,CAAC;AACrF,yBAAmB,KAAK,GAAG,gBAAgB;AAG3C,YAAM,kBAAkB,kCAAa;AAAA,QACnC,OAAO;AAAA,QACP,OAAO;AAAA,QACP,OAAO,iBAAiB;AAAA,QACxB;AAAA,MACF;AAGA,YAAM,UAAU,OAAO,OAAO,QAAQ,KAAK;AAC3C,cAAQ,SAAS,KAAK,GAAG,kBAAkB;AAC3C,cAAQ,SAAS,KAAK,GAAG,iBAAiB,WAAW,EAAE,iBAAiB;AAExE,YAAM,kBAAkB,KAAK,IAAI,KAAK;AAAA,QACpC;AAAA,QACA,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,kBAAkB,KAAK,uBAAuB,gBAAgB,IAAI,eAAe;AACvF,sBAAgB,WAAW,iBAAiB,eAAe;AAC3D,aAAO,gBAAgB,eAAe;AAEtC,WAAK,KAAK,iCAAkC,WAAW;AAAA,IACzD;AAEA,QAAI,WAAW;AACf,UAAM,OAAO,qBAAqB,EAAE,KAAK,MAAM;AAC7C,iBAAW;AAAA,IACb,CAAC;AACD,WAAO,CAAC,OAAO,sBAAsB;AACnC,YAAM,UAAU,OAAO,oBAAoB;AAC3C,YAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;AAClC,aAAO,OAAO,oBAAoB,QAAQ;AACxC,cAAM,SAAS,OAAO,oBAAoB,CAAC;AAC3C,aAAK,iBAAiB;AACtB,cAAM,KAAK,YAAY,MAAM;AAC7B,eAAO,oBAAoB,MAAM;AACjC,aAAK,iBAAiB;AAAA,MACxB;AAEA,aAAO,oBAAoB,QAAQ,MAAM,OAAO,oBAAoB,IAAI,CAAC;AACzE,UAAI,UAAU;AACZ,eAAO,yBAAyB;AAAA,MAClC;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,uBACE,UACA,QACiB;AACjB,UAAM,eAAe,IAAI,2CAAsB,2CAAsB;AACrE,iBAAa,GAAG,eAAe,CAAC,SAAS;AACvC,WAAK,wBAAwB,KAAK;AAClC,WAAK,MAAO,iBAAkB,qBAAqB;AAAA,QACjD,qBAAqB,KAAK,MAAO,iBAAkB;AAAA,QACnD,UAAU,KAAK,kBAAmB;AAAA,QAClC,UAAU,CAAC,IAAI;AAAA,MACjB,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,KAAK,cAAc;AACtB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAEA,QAAI,kBAAkB,sBAAW;AAC/B,eAAS,0BAA0B,UAAU,MAAM;AAAA,IACrD;AAEA,UAAM,WAAW;AACjB,QAAI,EAAE,OAAO,WAAW,WAAW;AAAA,IAEnC;AAEA,UAAM,YAAY,KAAK,MAAM,kBAAkB,MAAM,QAAQ;AAC7D,QAAI,CAAC,WAAW;AACd,YAAM,IAAI,MAAM,+DAA+D;AAAA,IACjF;AAEA,WAAO,KAAK,aAAa,WAAW,UAAU,WAAW,YAAY;AAAA,EACvE;AAAA,EAEA,MAAM,2BAA2B;AAC/B,QAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,oBAAoB;AAClE,WAAK,QACF,MAAM,EAAE,UAAU,KAAK,eAAe,GAAG,CAAC,EAC1C,MAAM,yEAAyE;AAClF;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,UAAI,KAAK,MAAM,uBAAuB,CAAC,KAAK,iBAAiB;AAC3D;AAAA,MACF;AACA,WAAK,sBAAsB;AAAA,IAC7B;AAEA,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAIA,QAAI,KAAK,iBAAiB,MAAM;AAC9B,uBAAiB,UAAU,KAAK,cAAc;AAC5C,YAAI,WAAW,mBAAmB,eAAgB;AAClD,YAAI,CAAC,OAAO,QAAS;AACrB,YAAI,OAAO,mBAAoB,QAAO,UAAU;AAAA,MAClD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,UAAU,KAAK,mBAAmB,GAAG,CAAC,EAAE,MAAM,uBAAuB;AAE1F,QAAI,KAAK,iBAAiB;AACxB,YAAM,sBAAsB,KAAK,IAAI,IAAI,KAAK;AAC9C,YAAM,qBAAqB,KAAK;AAAA,SAC7B,KAAK,4BAA4B,KAAK,KAAK;AAAA,QAC5C;AAAA,MACF;AACA,YAAM,UAA8B;AAAA,QAClC,WAAW,KAAK,IAAI;AAAA,QACpB,YAAY,KAAK,mBAAmB;AAAA,QACpC,qBAAqB;AAAA,QACrB;AAAA,MACF;AACA,WAAK,KAAK,2BAA4B,OAAO;AAAA,IAC/C;AAEA,SAAK,qBAAqB,KAAK,kBAAkB;AACjD,SAAK,qBAAqB;AAC1B,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,uBAAuB;AACrB,QACE,CAAC,KAAK,kBACN,CAAC,KAAK,eAAe,sBACrB,KAAK,eAAe,aACpB;AACA;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,sBAAsB,GAAG;AAGtC,YAAM,eAAe,KAAK,MAAM,cAAc,cAAc;AAAA,QAC1D,KAAK;AAAA,MACP;AACA,UAAI,aAAa,SAAS,KAAK,MAAM,mBAAmB;AACtD;AAAA,MACF;AAAA,IACF;AACA,SAAK,eAAe,UAAU;AAAA,EAChC;AAAA,EAEA,qBAAqB,QAAsB;AACzC,SAAK,aAAa,IAAI,MAAM;AAC5B,SAAK,aAAa,IAAI,mBAAmB,cAAc;AACvD,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA;AAAA,EAGA,MAAM,QAAQ;AAt+BhB;AAu+BI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AAEA,eAAK,UAAL,mBAAY,mBAAmB,0BAAU;AAAA,EAE3C;AACF;AAEA,gBAAgB,0BACd,UACA,QACuB;AAn/BzB;AAo/BE,QAAM,YAAY,KAAK,IAAI;AAC3B,MAAI,aAAa;AACjB,mBAAiB,SAAS,QAAQ;AAChC,UAAM,WAAU,WAAM,QAAQ,CAAC,MAAf,mBAAkB,MAAM;AACxC,QAAI,CAAC,QAAS;AAEd,QAAI,YAAY;AACd,mBAAa;AACb,0BAAI,EACD,MAAM,EAAE,UAAU,SAAS,KAAK,MAAM,KAAK,IAAI,IAAI,SAAS,EAAE,CAAC,EAC/D,MAAM,0BAA0B;AAAA,IACrC;AACA,UAAM;AAAA,EACR;AACF;AAGA,MAAM,wBAAwB;AAAA;AAAA,EAEnB,cAAc;AAAA,EACd,4BAA4B;AAAA,EAC5B,4BAA4B;AAAA;AAAA,EAC5B,0BAA0B;AAAA,EAEnC;AAAA,EACA;AAAA,EACA,oBAAoB,IAAI,oBAAO;AAAA,EAC/B,uBAAuB;AAAA,EACvB,2BAA2B;AAAA,EAC3B,YAAY;AAAA,EACZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,cACA,qBACA,OACA,cACA;AACA,SAAK,gBAAgB;AACrB,SAAK,oBAAoB;AACzB,SAAK,wBAAwB;AAC7B,SAAK,SAAS;AACd,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,CAAC,KAAK,kBAAkB;AAAA,EACjC;AAAA,EAEA,uBAAuB,YAAoB;AACzC,SAAK,uBAAuB,WAAW,KAAK;AAC5C,QAAI,KAAK,UAAW;AAEpB,UAAM,uBACJ,KAAK,IAAI,IAAI,KAAK,2BAA2B,KAAK;AACpD,QAAI,QAAQ,uBAAuB,KAAK,oBAAoB,KAAK;AACjE,YAAQ,KAAK,oBAAoB,IAAI,QAAQ,KAAK,4BAA4B;AAE9E,SAAK,KAAK,KAAK;AAAA,EACjB;AAAA;AAAA,EAGA,qBAAqB,GAAa;AAtjCpC;AAujCI,SAAK,YAAY;AACjB,QAAI,KAAK,YAAY;AACnB,iBAAK,WAAL,mBAAa;AAAA,IACf;AAAA,EACF;AAAA;AAAA,EAGA,mBAAmB,GAAa;AAC9B,SAAK,YAAY;AACjB,SAAK,2BAA2B,KAAK,IAAI;AAEzC,QAAI,KAAK,sBAAsB;AAC7B,YAAM,QAAQ,KAAK,oBAAoB,IACnC,KAAK,oBAAoB,KAAK,4BAC9B;AACJ,WAAK,KAAK,KAAK;AAAA,IACjB;AAAA,EACF;AAAA;AAAA,EAIA,sBAA+B;AAC7B,WACE,KAAK,qBAAqB,SAAS,KACnC,KAAK,YAAY,SAAS,KAAK,qBAAqB,KAAK,qBAAqB,SAAS,CAAC,CAAE;AAAA,EAE9F;AAAA,EAEA,eAAe;AACb,SAAK,uBAAuB;AAC5B,SAAK,2BAA2B;AAAA,EAClC;AAAA,EAEA,KAAK,OAAe;AAxlCtB;AAylCI,UAAM,UAAU,OAAOH,QAAe,SAAsB,WAAwB;AAClF,UAAI,KAAK,wBAAwB,CAAC,KAAK,aAAa,KAAK,eAAe;AACtE,cAAM,YAAY,KAAK,IAAI;AAC3B,cAAM,UAAU,MAAM,KAAK,cAAc,iBAAiB,OAAO;AACjE,cAAM,oBAAoB,KAAK,cAAc;AAC7C,cAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAI,UAAU,mBAAmB;AAC/B,UAAAA,SAAQ,KAAK;AAAA,QACf;AACA,QAAAA,SAAQ,KAAK,IAAI,GAAGA,SAAQ,OAAO;AAAA,MACrC;AACA,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,aAAa;AAClB,aAAK,cAAc;AAAA,MACrB,GAAGA,MAAK;AACR,aAAO,iBAAiB,SAAS,MAAM;AACrC,qBAAa,OAAO;AAAA,MACtB,CAAC;AAAA,IACH;AAEA,eAAK,WAAL,mBAAa;AACb,SAAK,SAAS,IAAI,gBAAgB;AAClC,SAAK,oBAAoB,IAAI,oBAAO;AACpC,UAAM,YAAY,KAAK,OAAO,QAAQ,KAAK;AAC3C,cAAU,OAAO,EAAE,MAAM,KAAK,OAAO,iBAAiB,MAAM,qBAAS,KAAK,CAAC;AAC3E,SAAK,qBAAqB,QAAQ,OAAO,WAAW,KAAK,OAAO,MAAM;AAAA,EACxE;AACF;","names":["import_llm","VPAEvent","BasicSentenceTokenizer","BasicWordTokenizer","EventEmitter","STTStreamAdapter","TTSStreamAdapter","participant","delay","resolve","isUsingTools","task"]}

package/dist/pipeline/pipeline_agent.d.ts

@@ -38,6 +38,11 @@ export type VPACallbacks = {
     [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;
     [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;
 };
+interface TurnDetector {
+    unlikelyThreshold: number;
+    supportsLanguage: (language?: string) => boolean;
+    predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;
+}
 export declare class AgentCallContext {
     #private;
     constructor(agent: VoicePipelineAgent, llmStream: LLMStream);
@@ -94,6 +99,8 @@ export interface VPAOptions {
     beforeTTSCallback: BeforeTTSCallback;
     /** Options for assistant transcription. */
     transcription: AgentTranscriptionOptions;
+    /** Turn detection model to use. */
+    turnDetector?: TurnDetector;
 }
 declare const VoicePipelineAgent_base: new () => TypedEmitter<VPACallbacks>;
 /** A pipeline agent (VAD + STT + LLM + TTS) implementation. */
@@ -102,6 +109,7 @@ export declare class VoicePipelineAgent extends VoicePipelineAgent_base {
     /** Minimum time played for the user speech to be committed to the chat context. */
     readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
     protected static readonly FLUSH_SENTINEL: unique symbol;
+    transcribedText: string;
     constructor(
     /** Voice Activity Detection instance. */
     vad: VAD, 

package/dist/pipeline/pipeline_agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline_agent.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline_agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAyB,iBAAiB,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAQxF,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,GAAG,EACJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAY,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAY,MAAM,iBAAiB,CAAC;AAErE,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,KAAK,GAAG,EAAsD,MAAM,iBAAiB,CAAC;AAM/F,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAG3C,OAAO,EAAE,KAAK,GAAG,EAA+B,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,YAAY,EAAmB,MAAM,mBAAmB,CAAC;AAIvE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAGtD,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,WAAW,KACjB,SAAS,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KACnC,YAAY,CAAC;AAElB,oBAAY,QAAQ;IAClB,qBAAqB,IAAA;IACrB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,sBAAsB,IAAA;IACtB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,wBAAwB,IAAA;IACxB,wBAAwB,IAAA;IACxB,uBAAuB,IAAA;IACvB,iBAAiB,IAAA;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7D,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC9D,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAChE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;IACzE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,IAAI,CAAC;IAC9E,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/D,CAAC;AAEF,qBAAa,gBAAgB;;gBAOf,KAAK,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS;IAM3D,MAAM,CAAC,UAAU,IAAI,gBAAgB;IAIrC,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG;IAIrC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,GAAe;IAInD,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,IAAI,iBAAiB,kBAEpB;IAED,mBAAmB,CAAC,OAAO,EAAE,WAAW;CAGzC;AAiBD,MAAM,WAAW,yBAAyB;IACxC,8DAA8D;IAC9D,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+DAA+D;IAC/D,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,aAAa,EAAE,aAAa,CAAC;IAC7B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC3C;AAWD,MAAM,WAAW,UAAU;IACzB,sCAAsC;IACtC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,4DAA4D;IAC5D,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+DAA+D;IAC/D,uBAAuB,EAAE,MAAM,CAAC;IAChC,sFAAsF;IACtF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6DAA6D;IAC7D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,mBAAmB,EAAE,OAAO,CAAC;IAS7B,iBAAiB,EAAE,iBAAiB,CAAC;IAQrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,2CAA2C;IAC3C,aAAa,EAAE,yBAAyB,CAAC;CAC1C;iDAgBkE,aAAa,YAAY,CAAC;AAD7F,+DAA+D;AAC/D,qBAAa,kBAAmB,SAAQ,uBAAsD;;IAC5F,mFAAmF;IACnF,QAAQ,CAAC,0BAA0B,OAAO;IAC1C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;;IA4BlE,yCAAyC;IACzC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,qCAAqC;IACrC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,6CAA6C;IAC7C,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAyB/C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,eAAe,EAE9B;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,iCAAiC;IACjC,KAAK;IACH,8BAA8B;IAC9B,IAAI,EAAE,IAAI;IACV;;;;;;OAMG;IACH,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW;IA8CvD,wDAAwD;IAClD,GAAG,CACP,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAClD,kBAAkB,UAAO,EACzB,YAAY,UAAO,GAClB,OAAO,CAAC,YAAY,CAAC;IA2iBxB,iCAAiC;IAC3B,KAAK;CAQZ"}
+{"version":3,"file":"pipeline_agent.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline_agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAyB,iBAAiB,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAQxF,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,GAAG,EACJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAY,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAY,MAAM,iBAAiB,CAAC;AAErE,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,KAAK,GAAG,EAAsD,MAAM,iBAAiB,CAAC;AAM/F,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEjF,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAG3C,OAAO,EAAE,KAAK,GAAG,EAA+B,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,YAAY,EAAmB,MAAM,mBAAmB,CAAC;AAIvE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAGtD,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,WAAW,KACjB,SAAS,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KACnC,YAAY,CAAC;AAElB,oBAAY,QAAQ;IAClB,qBAAqB,IAAA;IACrB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,sBAAsB,IAAA;IACtB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,wBAAwB,IAAA;IACxB,wBAAwB,IAAA;IACxB,uBAAuB,IAAA;IACvB,iBAAiB,IAAA;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7D,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC9D,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAChE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;IACzE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,IAAI,CAAC;IAC9E,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/D,CAAC;AAEF,UAAU,YAAY;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC;IACjD,gBAAgB,EAAE,CAAC,OAAO,EAAE,WAAW,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAC7D;AAED,qBAAa,gBAAgB;;gBAOf,KAAK,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS;IAM3D,MAAM,CAAC,UAAU,IAAI,gBAAgB;IAIrC,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG;IAIrC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,GAAe;IAInD,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,IAAI,iBAAiB,kBAEpB;IAED,mBAAmB,CAAC,OAAO,EAAE,WAAW;CAGzC;AAiBD,MAAM,WAAW,yBAAyB;IACxC,8DAA8D;IAC9D,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+DAA+D;IAC/D,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,aAAa,EAAE,aAAa,CAAC;IAC7B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC3C;AAWD,MAAM,WAAW,UAAU;IACzB,sCAAsC;IACtC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,4DAA4D;IAC5D,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+DAA+D;IAC/D,uBAAuB,EAAE,MAAM,CAAC;IAChC,sFAAsF;IACtF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6DAA6D;IAC7D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,mBAAmB,EAAE,OAAO,CAAC;IAS7B,iBAAiB,EAAE,iBAAiB,CAAC;IAQrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,2CAA2C;IAC3C,aAAa,EAAE,yBAAyB,CAAC;IACzC,mCAAmC;IACnC,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;iDAgBkE,aAAa,YAAY,CAAC;AAD7F,+DAA+D;AAC/D,qBAAa,kBAAmB,SAAQ,uBAAsD;;IAC5F,mFAAmF;IACnF,QAAQ,CAAC,0BAA0B,OAAO;IAC1C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IAapE,eAAe,SAAM;;IAiBnB,yCAAyC;IACzC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,qCAAqC;IACrC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,6CAA6C;IAC7C,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IA2B/C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,eAAe,EAE9B;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,iCAAiC;IACjC,KAAK;IACH,8BAA8B;IAC9B,IAAI,EAAE,IAAI;IACV;;;;;;OAMG;IACH,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW;IA8CvD,wDAAwD;IAClD,GAAG,CACP,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAClD,kBAAkB,UAAO,EACzB,YAAY,UAAO,GAClB,OAAO,CAAC,YAAY,CAAC;IA4lBxB,iCAAiC;IAC3B,KAAK;CAQZ"}

package/dist/pipeline/pipeline_agent.js

@@ -5,6 +5,7 @@ import {
   TrackPublishOptions,
   TrackSource
 } from "@livekit/rtc-node";
+import { randomUUID } from "node:crypto";
 import EventEmitter from "node:events";
 import { LLMEvent, LLMStream } from "../llm/index.js";
 import { ChatContext, ChatMessage, ChatRole } from "../llm/index.js";
@@ -15,6 +16,7 @@ import {
   WordTokenizer as BasicWordTokenizer,
   hyphenateWord
 } from "../tokenize/basic/index.js";
+import { TextAudioSynchronizer, defaultTextSyncOptions } from "../transcription.js";
 import { TTSEvent, StreamAdapter as TTSStreamAdapter } from "../tts/index.js";
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from "../utils.js";
 import { VADEventType } from "../vad.js";
@@ -111,7 +113,7 @@ class VoicePipelineAgent extends EventEmitter {
   #pendingAgentReply;
   #agentReplyTask;
   #playingSpeech;
-  #transcribedText = "";
+  transcribedText = "";
   #transcribedInterimText = "";
   #speechQueueOpen = new Future();
   #speechQueue = new AsyncIterableQueue();
@@ -124,6 +126,8 @@ class VoicePipelineAgent extends EventEmitter {
   #agentPublication;
   #lastFinalTranscriptTime;
   #lastSpeechTime;
+  #transcriptionId;
+  #agentTranscribedText = "";
   constructor(vad, stt, llm, tts, opts = defaultVPAOptions) {
     super();
     this.#opts = { ...defaultVPAOptions, ...opts };
@@ -139,7 +143,9 @@ class VoicePipelineAgent extends EventEmitter {
     this.#tts = tts;
     this.#deferredValidation = new DeferredReplyValidation(
       this.#validateReplyIfPossible.bind(this),
-      this.#opts.minEndpointingDelay
+      this.#opts.minEndpointingDelay,
+      this,
+      this.#opts.turnDetector
     );
   }
   get fncCtx() {
@@ -301,13 +307,48 @@ class VoicePipelineAgent extends EventEmitter {
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
     this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
       this.#transcribedInterimText = event.alternatives[0].text;
+      this.#room.localParticipant.publishTranscription({
+        participantIdentity: this.#humanInput.participant.identity,
+        trackSid: this.#humanInput.subscribedTrack.sid,
+        segments: [
+          {
+            text: this.#transcribedInterimText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: ""
+          }
+        ]
+      });
     });
     this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
       const newTranscript = event.alternatives[0].text;
       if (!newTranscript) return;
+      if (!this.#transcriptionId) {
+        this.#transcriptionId = randomUUID();
+      }
       this.#lastFinalTranscriptTime = Date.now();
-      this.#transcribedText += (this.#transcribedText ? " " : "") + newTranscript;
+      this.transcribedText += (this.transcribedText ? " " : "") + newTranscript;
+      this.#room.localParticipant.publishTranscription({
+        participantIdentity: this.#humanInput.participant.identity,
+        trackSid: this.#humanInput.subscribedTrack.sid,
+        segments: [
+          {
+            text: this.transcribedText,
+            id: this.#transcriptionId,
+            final: true,
+            startTime: BigInt(0),
+            endTime: BigInt(0),
+            language: ""
+          }
+        ]
+      });
+      this.#transcriptionId = void 0;
       if (this.#opts.preemptiveSynthesis && (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
         this.#synthesizeAgentReply();
       }
@@ -358,7 +399,7 @@ class VoicePipelineAgent extends EventEmitter {
     this.#pendingAgentReply = SpeechHandle.createAssistantReply(
       this.#opts.allowInterruptions,
       true,
-      this.#transcribedText
+      this.transcribedText
     );
     const newHandle = this.#pendingAgentReply;
     this.#agentReplyTask = this.#synthesizeAnswerTask(this.#agentReplyTask, newHandle);
@@ -435,7 +476,7 @@ class VoicePipelineAgent extends EventEmitter {
       const userMsg = ChatMessage.create({ text: userQuestion, role: ChatRole.USER });
       this.chatCtx.messages.push(userMsg);
       this.emit(4 /* USER_SPEECH_COMMITTED */, userMsg);
-      this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+      this.transcribedText = this.transcribedText.slice(userQuestion.length);
       handle.markUserCommitted();
     };
     commitUserQuestionIfNeeded();
@@ -449,7 +490,7 @@ class VoicePipelineAgent extends EventEmitter {
       if (handle.interrupted) break;
     }
     commitUserQuestionIfNeeded();
-    const collectedText = handle.synthesisHandle.text;
+    let collectedText = this.#agentTranscribedText;
     const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
     const interrupted = handle.interrupted;
     if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
@@ -457,7 +498,7 @@ class VoicePipelineAgent extends EventEmitter {
         this.chatCtx.messages.push(...handle.extraToolsMessages);
       }
       if (interrupted) {
-        collectedText + "\u2026";
+        collectedText += "\u2026";
       }
       const msg = ChatMessage.create({ text: collectedText, role: ChatRole.ASSISTANT });
       this.chatCtx.messages.push(msg);
@@ -552,6 +593,15 @@ class VoicePipelineAgent extends EventEmitter {
     handle.setDone();
   }
   #synthesizeAgentSpeech(speechId, source) {
+    const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
+    synchronizer.on("textUpdated", (text) => {
+      this.#agentTranscribedText = text.text;
+      this.#room.localParticipant.publishTranscription({
+        participantIdentity: this.#room.localParticipant.identity,
+        trackSid: this.#agentPublication.sid,
+        segments: [text]
+      });
+    });
     if (!this.#agentOutput) {
       throw new Error("agent output should be initialized when ready");
     }
@@ -565,7 +615,7 @@ class VoicePipelineAgent extends EventEmitter {
     if (!ttsSource) {
       throw new Error("beforeTTSCallback must return string or AsyncIterable<string>");
     }
-    return this.#agentOutput.synthesize(speechId, ttsSource);
+    return this.#agentOutput.synthesize(speechId, ttsSource, synchronizer);
   }
   async #validateReplyIfPossible() {
     if (this.#playingSpeech && !this.#playingSpeech.allowInterruptions) {
@@ -573,7 +623,7 @@ class VoicePipelineAgent extends EventEmitter {
       return;
     }
     if (!this.#pendingAgentReply) {
-      if (this.#opts.preemptiveSynthesis || !this.#transcribedText) {
+      if (this.#opts.preemptiveSynthesis || !this.transcribedText) {
         return;
       }
       this.#synthesizeAgentReply();
@@ -655,6 +705,7 @@ class DeferredReplyValidation {
   PUNCTUATION_REDUCE_FACTOR = 0.75;
   LATE_TRANSCRIPT_TOLERANCE = 1.5;
   // late compared to end of speech
+  UNLIKELY_ENDPOINT_DELAY = 6e3;
   #validateFunc;
   #validatingPromise;
   #validatingFuture = new Future();
@@ -663,11 +714,15 @@ class DeferredReplyValidation {
   #speaking = false;
   #endOfSpeechDelay;
   #finalTranscriptDelay;
+  #turnDetector;
+  #agent;
   #abort;
-  constructor(validateFunc, minEndpointingDelay) {
+  constructor(validateFunc, minEndpointingDelay, agent, turnDetector) {
     this.#validateFunc = validateFunc;
     this.#endOfSpeechDelay = minEndpointingDelay;
     this.#finalTranscriptDelay = minEndpointingDelay;
+    this.#agent = agent;
+    this.#turnDetector = turnDetector;
   }
   get validating() {
     return !this.#validatingFuture.done;
@@ -707,7 +762,17 @@ class DeferredReplyValidation {
   }
   #run(delay) {
     var _a;
-    const runTask = async (delay2, signal) => {
+    const runTask = async (delay2, chatCtx, signal) => {
+      if (this.#lastFinalTranscript && !this.#speaking && this.#turnDetector) {
+        const startTime = Date.now();
+        const eotProb = await this.#turnDetector.predictEndOfTurn(chatCtx);
+        const unlikelyThreshold = this.#turnDetector.unlikelyThreshold;
+        const elapsed = Date.now() - startTime;
+        if (eotProb < unlikelyThreshold) {
+          delay2 = this.UNLIKELY_ENDPOINT_DELAY;
+        }
+        delay2 = Math.max(0, delay2 - elapsed);
+      }
       const timeout = setTimeout(() => {
         this.#resetStates();
         this.#validateFunc();
@@ -719,7 +784,9 @@ class DeferredReplyValidation {
     (_a = this.#abort) == null ? void 0 : _a.abort();
     this.#abort = new AbortController();
     this.#validatingFuture = new Future();
-    this.#validatingPromise = runTask(delay, this.#abort.signal);
+    const detectCtx = this.#agent.chatCtx.copy();
+    detectCtx.append({ text: this.#agent.transcribedText, role: ChatRole.USER });
+    this.#validatingPromise = runTask(delay, detectCtx, this.#abort.signal);
   }
 }
 export {