@livekit/agents 0.7.8 → 1.0.0-next.0

package/dist/voice/transcription/synchronizer.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/voice/transcription/synchronizer.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { delay } from '@std/async';\nimport type { ReadableStream, WritableStreamDefaultWriter } from 'node:stream/web';\nimport { log } from '../../log.js';\nimport { IdentityTransform } from '../../stream/identity_transform.js';\nimport type { SentenceStream, SentenceTokenizer } from '../../tokenize/index.js';\nimport { basic } from '../../tokenize/index.js';\nimport { Future, Task } from '../../utils.js';\nimport { AudioOutput, type PlaybackFinishedEvent, TextOutput } from '../io.js';\n\nconst STANDARD_SPEECH_RATE = 3.83; // hyphens (syllables) per second\n\ninterface TextSyncOptions {\n  speed: number;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n  sentenceTokenizer: SentenceTokenizer;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedText: string;\n}\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\nclass SegmentSynchronizerImpl {\n  private textData: TextData;\n  private audioData: AudioData;\n  private speed: number;\n  private outputStream: IdentityTransform<string>;\n  private outputStreamWriter: WritableStreamDefaultWriter<string>;\n  private captureTask: Promise<void>;\n  private startWallTime?: number;\n\n  private startFuture: Future = new Future();\n  private closedFuture: Future = new Future();\n  private playbackCompleted: boolean = false;\n\n  private logger = log();\n\n  constructor(\n    private readonly options: TextSyncOptions,\n    private readonly nextInChain: TextOutput,\n  ) {\n    this.speed = options.speed * STANDARD_SPEECH_RATE; // hyphens per second\n    this.textData = {\n      sentenceStream: options.sentenceTokenizer.stream(),\n      pushedText: '',\n      done: false,\n      forwardedHyphens: 0,\n      forwardedText: '',\n    };\n    this.audioData = {\n      pushedDuration: 0,\n      done: false,\n    };\n    this.outputStream = new IdentityTransform();\n    this.outputStreamWriter = this.outputStream.writable.getWriter();\n\n    this.mainTask()\n      .then(() => {\n        this.outputStreamWriter.close();\n      })\n      .catch((error) => {\n        this.logger.error({ error }, 'mainTask SegmentSynchronizerImpl');\n      });\n    this.captureTask = this.captureTaskImpl();\n  }\n\n  get closed() {\n    return this.closedFuture.done;\n  }\n\n  get audioInputEnded() {\n    return this.audioData.done;\n  }\n\n  get textInputEnded() {\n    return this.textData.done;\n  }\n\n  get readable(): ReadableStream<string> {\n    return this.outputStream.readable;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.pushAudio called after close');\n      return;\n    }\n    // TODO(AJS-102): use frame.durationMs once available in rtc-node\n    const frameDuration = frame.samplesPerChannel / frame.sampleRate;\n\n    if (!this.startWallTime && frameDuration > 0) {\n      this.startWallTime = Date.now();\n      this.startFuture.resolve();\n    }\n\n    this.audioData.pushedDuration += frameDuration;\n  }\n\n  endAudioInput() {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.endAudioInput called after close');\n      return;\n    }\n\n    this.audioData.done = true;\n  }\n\n  pushText(text: string) {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.pushText called after close');\n      return;\n    }\n\n    this.textData.sentenceStream.pushText(text);\n    this.textData.pushedText += text;\n  }\n\n  endTextInput() {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.endTextInput called after close');\n      return;\n    }\n\n    this.textData.done = true;\n    this.textData.sentenceStream.endInput();\n  }\n\n  markPlaybackFinished(_playbackPosition: number, interrupted: boolean) {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.markPlaybackFinished called after close');\n      return;\n    }\n\n    if (!this.textData.done || !this.audioData.done) {\n      this.logger.warn(\n        { textDone: this.textData.done, audioDone: this.audioData.done },\n        'SegmentSynchronizerImpl.markPlaybackFinished called before text/audio input is done',\n      );\n      return;\n    }\n\n    if (!interrupted) {\n      this.playbackCompleted = true;\n    }\n  }\n\n  get synchronizedTranscript(): string {\n    if (this.playbackCompleted) {\n      return this.textData.pushedText;\n    }\n    return this.textData.forwardedText;\n  }\n\n  private async captureTaskImpl() {\n    // Don't use a for-await loop here, because exiting the loop will close the writer in the\n    // outputStream, which will cause an error in the mainTask.then method.\n    const reader = this.outputStream.readable.getReader();\n    while (true) {\n      const { done, value: text } = await reader.read();\n      if (done) {\n        break;\n      }\n      this.textData.forwardedText += text;\n      await this.nextInChain.captureText(text);\n    }\n    reader.releaseLock();\n    this.nextInChain.flush();\n  }\n\n  private async mainTask(): Promise<void> {\n    await this.startFuture.await;\n\n    if (this.closed && !this.playbackCompleted) {\n      return;\n    }\n\n    if (!this.startWallTime) {\n      throw new Error('startWallTime is not set when starting SegmentSynchronizerImpl.mainTask');\n    }\n\n    for await (const textSegment of this.textData.sentenceStream) {\n      const sentence = textSegment.token;\n\n      let textCursor = 0;\n      if (this.closed && !this.playbackCompleted) {\n        return;\n      }\n\n      for (const [word, _, endPos] of this.options.splitWords(sentence)) {\n        if (this.closed && !this.playbackCompleted) {\n          return;\n        }\n\n        if (this.playbackCompleted) {\n          this.outputStreamWriter.write(sentence.slice(textCursor, endPos));\n          textCursor = endPos;\n          continue;\n        }\n\n        const wordHphens = this.options.hyphenateWord(word).length;\n        const elapsedSeconds = (Date.now() - this.startWallTime) / 1000;\n        const targetHyphens = elapsedSeconds * this.options.speed;\n        const hyphensBehind = Math.max(0, targetHyphens - this.textData.forwardedHyphens);\n        let delay = Math.max(0, wordHphens - hyphensBehind) / this.speed;\n\n        if (this.playbackCompleted) {\n          delay = 0;\n        }\n\n        await this.sleepIfNotClosed(delay / 2);\n        this.outputStreamWriter.write(sentence.slice(textCursor, endPos));\n        await this.sleepIfNotClosed(delay / 2);\n\n        this.textData.forwardedHyphens += wordHphens;\n        textCursor = endPos;\n      }\n\n      if (textCursor < sentence.length) {\n        const remaining = sentence.slice(textCursor);\n        this.outputStreamWriter.write(remaining);\n      }\n    }\n  }\n\n  private async sleepIfNotClosed(sleepTimeSeconds: number) {\n    if (this.closed) {\n      return;\n    }\n    await delay(sleepTimeSeconds * 1000);\n  }\n\n  async close(): Promise<void> {\n    if (this.closed) {\n      return;\n    }\n    this.closedFuture.resolve();\n    this.startFuture.resolve(); // avoid deadlock of mainTaskImpl in case it never started\n    this.textData.sentenceStream.close();\n    await this.captureTask;\n  }\n}\n\nexport interface TranscriptionSynchronizerOptions {\n  speed: number;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n  sentenceTokenizer: SentenceTokenizer;\n}\n\nexport const defaultTextSyncOptions: TranscriptionSynchronizerOptions = {\n  speed: 1,\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n  sentenceTokenizer: new basic.SentenceTokenizer({\n    retainFormat: true,\n  }),\n};\n\nexport class TranscriptionSynchronizer {\n  readonly audioOutput: SyncedAudioOutput;\n  readonly textOutput: SyncedTextOutput;\n\n  private options: TextSyncOptions;\n  private rotateSegmentTask: Task<void>;\n  private _enabled: boolean = true;\n  private closed: boolean = false;\n\n  /* @internal */\n  _impl: SegmentSynchronizerImpl;\n\n  private logger = log();\n\n  constructor(\n    nextInChainAudio: AudioOutput,\n    nextInChainText: TextOutput,\n    options: TranscriptionSynchronizerOptions = defaultTextSyncOptions,\n  ) {\n    this.audioOutput = new SyncedAudioOutput(this, nextInChainAudio);\n    this.textOutput = new SyncedTextOutput(this, nextInChainText);\n    this.options = {\n      speed: options.speed,\n      hyphenateWord: options.hyphenateWord,\n      splitWords: options.splitWords,\n      sentenceTokenizer: options.sentenceTokenizer,\n    };\n\n    // initial segment/first segment, recreated for each new segment\n    this._impl = new SegmentSynchronizerImpl(this.options, nextInChainText);\n    this.rotateSegmentTask = Task.from((controller) =>\n      this.rotateSegmentTaskImpl(controller.signal),\n    );\n  }\n\n  get enabled(): boolean {\n    return this._enabled;\n  }\n\n  set enabled(enabled: boolean) {\n    if (this._enabled === enabled) {\n      return;\n    }\n\n    this._enabled = enabled;\n    this.rotateSegment();\n  }\n\n  rotateSegment() {\n    if (this.closed) {\n      return;\n    }\n\n    if (!this.rotateSegmentTask.done) {\n      this.logger.warn('rotateSegment called while previous segment is still being rotated');\n    }\n    this.rotateSegmentTask = Task.from((controller) =>\n      this.rotateSegmentTaskImpl(controller.signal, this.rotateSegmentTask),\n    );\n  }\n\n  async close(): Promise<void> {\n    this.closed = true;\n    await this.rotateSegmentTask.cancelAndWait();\n    await this._impl.close();\n  }\n\n  async barrier(): Promise<void> {\n    if (this.rotateSegmentTask.done) {\n      return;\n    }\n    await this.rotateSegmentTask.result;\n  }\n\n  private async rotateSegmentTaskImpl(abort: AbortSignal, oldTask?: Task<void>) {\n    if (oldTask) {\n      await oldTask.result;\n    }\n\n    if (abort.aborted) {\n      return;\n    }\n    await this._impl.close();\n    this._impl = new SegmentSynchronizerImpl(this.options, this.textOutput.nextInChain);\n  }\n}\n\nclass SyncedAudioOutput extends AudioOutput {\n  private pushedDuration: number = 0.0;\n\n  constructor(\n    public synchronizer: TranscriptionSynchronizer,\n    private nextInChainAudio: AudioOutput,\n  ) {\n    super(nextInChainAudio.sampleRate, nextInChainAudio);\n  }\n\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    // using barrier() on capture should be sufficient, flush() must not be called if\n    // capture_frame isn't completed\n    await this.synchronizer.barrier();\n\n    await super.captureFrame(frame);\n    await this.nextInChainAudio.captureFrame(frame); // passthrough audio\n\n    // TODO(AJS-102): use frame.durationMs once available in rtc-node\n    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n\n    if (!this.synchronizer.enabled) {\n      return;\n    }\n\n    if (this.synchronizer._impl.audioInputEnded) {\n      this.logger.warn(\n        'SegmentSynchronizerImpl audio marked as ended in capture audio, rotating segment',\n      );\n      this.synchronizer.rotateSegment();\n      await this.synchronizer.barrier();\n    }\n    this.synchronizer._impl.pushAudio(frame);\n  }\n\n  flush() {\n    super.flush();\n    this.nextInChainAudio.flush();\n\n    if (!this.synchronizer.enabled) {\n      return;\n    }\n\n    if (!this.pushedDuration) {\n      // in case there is no audio after the text was pushed, rotate the segment\n      this.synchronizer.rotateSegment();\n      return;\n    }\n\n    this.synchronizer._impl.endAudioInput();\n  }\n\n  clearBuffer() {\n    this.nextInChainAudio.clearBuffer();\n  }\n\n  // this is going to be automatically called by the next_in_chain\n  onPlaybackFinished(ev: PlaybackFinishedEvent) {\n    if (!this.synchronizer.enabled) {\n      super.onPlaybackFinished(ev);\n      return;\n    }\n\n    this.synchronizer._impl.markPlaybackFinished(ev.playbackPosition, ev.interrupted);\n    super.onPlaybackFinished({\n      playbackPosition: ev.playbackPosition,\n      interrupted: ev.interrupted,\n      synchronizedTranscript: this.synchronizer._impl.synchronizedTranscript,\n    });\n\n    this.synchronizer.rotateSegment();\n    this.pushedDuration = 0.0;\n  }\n}\n\nclass SyncedTextOutput extends TextOutput {\n  private capturing: boolean = false;\n  private logger = log();\n\n  constructor(\n    private readonly synchronizer: TranscriptionSynchronizer,\n    public readonly nextInChain: TextOutput,\n  ) {\n    super(nextInChain);\n  }\n\n  async captureText(text: string): Promise<void> {\n    await this.synchronizer.barrier();\n\n    if (!this.synchronizer.enabled) {\n      // pass through to the next in chain\n      await this.nextInChain.captureText(text);\n      return;\n    }\n\n    this.capturing = true;\n    if (this.synchronizer._impl.textInputEnded) {\n      this.logger.warn(\n        'SegmentSynchronizerImpl text marked as ended in capture text, rotating segment',\n      );\n      this.synchronizer.rotateSegment();\n      await this.synchronizer.barrier();\n    }\n    this.synchronizer._impl.pushText(text);\n  }\n\n  flush() {\n    if (!this.synchronizer.enabled) {\n      this.nextInChain.flush(); // passthrough text if the synchronizer is disabled\n      return;\n    }\n\n    if (!this.capturing) {\n      return;\n    }\n\n    this.capturing = false;\n    this.synchronizer._impl.endTextInput();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAsB;AAEtB,iBAAoB;AACpB,gCAAkC;AAElC,sBAAsB;AACtB,mBAA6B;AAC7B,gBAAoE;AAEpE,MAAM,uBAAuB;AAsB7B,MAAM,wBAAwB;AAAA,EAe5B,YACmB,SACA,aACjB;AAFiB;AACA;AAEjB,SAAK,QAAQ,QAAQ,QAAQ;AAC7B,SAAK,WAAW;AAAA,MACd,gBAAgB,QAAQ,kBAAkB,OAAO;AAAA,MACjD,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,kBAAkB;AAAA,MAClB,eAAe;AAAA,IACjB;AACA,SAAK,YAAY;AAAA,MACf,gBAAgB;AAAA,MAChB,MAAM;AAAA,IACR;AACA,SAAK,eAAe,IAAI,4CAAkB;AAC1C,SAAK,qBAAqB,KAAK,aAAa,SAAS,UAAU;AAE/D,SAAK,SAAS,EACX,KAAK,MAAM;AACV,WAAK,mBAAmB,MAAM;AAAA,IAChC,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,WAAK,OAAO,MAAM,EAAE,MAAM,GAAG,kCAAkC;AAAA,IACjE,CAAC;AACH,SAAK,cAAc,KAAK,gBAAgB;AAAA,EAC1C;AAAA,EAzCQ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,cAAsB,IAAI,oBAAO;AAAA,EACjC,eAAuB,IAAI,oBAAO;AAAA,EAClC,oBAA6B;AAAA,EAE7B,aAAS,gBAAI;AAAA,EA+BrB,IAAI,SAAS;AACX,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,IAAI,kBAAkB;AACpB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA,EAEA,IAAI,iBAAiB;AACnB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,IAAI,WAAmC;AACrC,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,UAAU,OAAmB;AAC3B,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,sDAAsD;AACvE;AAAA,IACF;AAEA,UAAM,gBAAgB,MAAM,oBAAoB,MAAM;AAEtD,QAAI,CAAC,KAAK,iBAAiB,gBAAgB,GAAG;AAC5C,WAAK,gBAAgB,KAAK,IAAI;AAC9B,WAAK,YAAY,QAAQ;AAAA,IAC3B;AAEA,SAAK,UAAU,kBAAkB;AAAA,EACnC;AAAA,EAEA,gBAAgB;AACd,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,0DAA0D;AAC3E;AAAA,IACF;AAEA,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA,EAEA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,qDAAqD;AACtE;AAAA,IACF;AAEA,SAAK,SAAS,eAAe,SAAS,IAAI;AAC1C,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,eAAe;AACb,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,yDAAyD;AAC1E;AAAA,IACF;AAEA,SAAK,SAAS,OAAO;AACrB,SAAK,SAAS,eAAe,SAAS;AAAA,EACxC;AAAA,EAEA,qBAAqB,mBAA2B,aAAsB;AACpE,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,iEAAiE;AAClF;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,SAAS,QAAQ,CAAC,KAAK,UAAU,MAAM;AAC/C,WAAK,OAAO;AAAA,QACV,EAAE,UAAU,KAAK,SAAS,MAAM,WAAW,KAAK,UAAU,KAAK;AAAA,QAC/D;AAAA,MACF;AACA;AAAA,IACF;AAEA,QAAI,CAAC,aAAa;AAChB,WAAK,oBAAoB;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,IAAI,yBAAiC;AACnC,QAAI,KAAK,mBAAmB;AAC1B,aAAO,KAAK,SAAS;AAAA,IACvB;AACA,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,MAAc,kBAAkB;AAG9B,UAAM,SAAS,KAAK,aAAa,SAAS,UAAU;AACpD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,OAAO,KAAK,IAAI,MAAM,OAAO,KAAK;AAChD,UAAI,MAAM;AACR;AAAA,MACF;AACA,WAAK,SAAS,iBAAiB;AAC/B,YAAM,KAAK,YAAY,YAAY,IAAI;AAAA,IACzC;AACA,WAAO,YAAY;AACnB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA,EAEA,MAAc,WAA0B;AACtC,UAAM,KAAK,YAAY;AAEvB,QAAI,KAAK,UAAU,CAAC,KAAK,mBAAmB;AAC1C;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,eAAe;AACvB,YAAM,IAAI,MAAM,yEAAyE;AAAA,IAC3F;AAEA,qBAAiB,eAAe,KAAK,SAAS,gBAAgB;AAC5D,YAAM,WAAW,YAAY;AAE7B,UAAI,aAAa;AACjB,UAAI,KAAK,UAAU,CAAC,KAAK,mBAAmB;AAC1C;AAAA,MACF;AAEA,iBAAW,CAAC,MAAM,GAAG,MAAM,KAAK,KAAK,QAAQ,WAAW,QAAQ,GAAG;AACjE,YAAI,KAAK,UAAU,CAAC,KAAK,mBAAmB;AAC1C;AAAA,QACF;AAEA,YAAI,KAAK,mBAAmB;AAC1B,eAAK,mBAAmB,MAAM,SAAS,MAAM,YAAY,MAAM,CAAC;AAChE,uBAAa;AACb;AAAA,QACF;AAEA,cAAM,aAAa,KAAK,QAAQ,cAAc,IAAI,EAAE;AACpD,cAAM,kBAAkB,KAAK,IAAI,IAAI,KAAK,iBAAiB;AAC3D,cAAM,gBAAgB,iBAAiB,KAAK,QAAQ;AACpD,cAAM,gBAAgB,KAAK,IAAI,GAAG,gBAAgB,KAAK,SAAS,gBAAgB;AAChF,YAAIA,SAAQ,KAAK,IAAI,GAAG,aAAa,aAAa,IAAI,KAAK;AAE3D,YAAI,KAAK,mBAAmB;AAC1B,UAAAA,SAAQ;AAAA,QACV;AAEA,cAAM,KAAK,iBAAiBA,SAAQ,CAAC;AACrC,aAAK,mBAAmB,MAAM,SAAS,MAAM,YAAY,MAAM,CAAC;AAChE,cAAM,KAAK,iBAAiBA,SAAQ,CAAC;AAErC,aAAK,SAAS,oBAAoB;AAClC,qBAAa;AAAA,MACf;AAEA,UAAI,aAAa,SAAS,QAAQ;AAChC,cAAM,YAAY,SAAS,MAAM,UAAU;AAC3C,aAAK,mBAAmB,MAAM,SAAS;AAAA,MACzC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBAAiB,kBAA0B;AACvD,QAAI,KAAK,QAAQ;AACf;AAAA,IACF;AACA,cAAM,oBAAM,mBAAmB,GAAI;AAAA,EACrC;AAAA,EAEA,MAAM,QAAuB;AAC3B,QAAI,KAAK,QAAQ;AACf;AAAA,IACF;AACA,SAAK,aAAa,QAAQ;AAC1B,SAAK,YAAY,QAAQ;AACzB,SAAK,SAAS,eAAe,MAAM;AACnC,UAAM,KAAK;AAAA,EACb;AACF;AASO,MAAM,yBAA2D;AAAA,EACtE,OAAO;AAAA,EACP,eAAe,sBAAM;AAAA,EACrB,YAAY,sBAAM;AAAA,EAClB,mBAAmB,IAAI,sBAAM,kBAAkB;AAAA,IAC7C,cAAc;AAAA,EAChB,CAAC;AACH;AAEO,MAAM,0BAA0B;AAAA,EAC5B;AAAA,EACA;AAAA,EAED;AAAA,EACA;AAAA,EACA,WAAoB;AAAA,EACpB,SAAkB;AAAA;AAAA,EAG1B;AAAA,EAEQ,aAAS,gBAAI;AAAA,EAErB,YACE,kBACA,iBACA,UAA4C,wBAC5C;AACA,SAAK,cAAc,IAAI,kBAAkB,MAAM,gBAAgB;AAC/D,SAAK,aAAa,IAAI,iBAAiB,MAAM,eAAe;AAC5D,SAAK,UAAU;AAAA,MACb,OAAO,QAAQ;AAAA,MACf,eAAe,QAAQ;AAAA,MACvB,YAAY,QAAQ;AAAA,MACpB,mBAAmB,QAAQ;AAAA,IAC7B;AAGA,SAAK,QAAQ,IAAI,wBAAwB,KAAK,SAAS,eAAe;AACtE,SAAK,oBAAoB,kBAAK;AAAA,MAAK,CAAC,eAClC,KAAK,sBAAsB,WAAW,MAAM;AAAA,IAC9C;AAAA,EACF;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAAQ,SAAkB;AAC5B,QAAI,KAAK,aAAa,SAAS;AAC7B;AAAA,IACF;AAEA,SAAK,WAAW;AAChB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,gBAAgB;AACd,QAAI,KAAK,QAAQ;AACf;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,kBAAkB,MAAM;AAChC,WAAK,OAAO,KAAK,oEAAoE;AAAA,IACvF;AACA,SAAK,oBAAoB,kBAAK;AAAA,MAAK,CAAC,eAClC,KAAK,sBAAsB,WAAW,QAAQ,KAAK,iBAAiB;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,MAAM,QAAuB;AAC3B,SAAK,SAAS;AACd,UAAM,KAAK,kBAAkB,cAAc;AAC3C,UAAM,KAAK,MAAM,MAAM;AAAA,EACzB;AAAA,EAEA,MAAM,UAAyB;AAC7B,QAAI,KAAK,kBAAkB,MAAM;AAC/B;AAAA,IACF;AACA,UAAM,KAAK,kBAAkB;AAAA,EAC/B;AAAA,EAEA,MAAc,sBAAsB,OAAoB,SAAsB;AAC5E,QAAI,SAAS;AACX,YAAM,QAAQ;AAAA,IAChB;AAEA,QAAI,MAAM,SAAS;AACjB;AAAA,IACF;AACA,UAAM,KAAK,MAAM,MAAM;AACvB,SAAK,QAAQ,IAAI,wBAAwB,KAAK,SAAS,KAAK,WAAW,WAAW;AAAA,EACpF;AACF;AAEA,MAAM,0BAA0B,sBAAY;AAAA,EAG1C,YACS,cACC,kBACR;AACA,UAAM,iBAAiB,YAAY,gBAAgB;AAH5C;AACC;AAAA,EAGV;AAAA,EAPQ,iBAAyB;AAAA,EASjC,MAAM,aAAa,OAAkC;AAGnD,UAAM,KAAK,aAAa,QAAQ;AAEhC,UAAM,MAAM,aAAa,KAAK;AAC9B,UAAM,KAAK,iBAAiB,aAAa,KAAK;AAG9C,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AAEvD,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,MAAM,iBAAiB;AAC3C,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AACA,WAAK,aAAa,cAAc;AAChC,YAAM,KAAK,aAAa,QAAQ;AAAA,IAClC;AACA,SAAK,aAAa,MAAM,UAAU,KAAK;AAAA,EACzC;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,iBAAiB,MAAM;AAE5B,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AAExB,WAAK,aAAa,cAAc;AAChC;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,cAAc;AAAA,EACxC;AAAA,EAEA,cAAc;AACZ,SAAK,iBAAiB,YAAY;AAAA,EACpC;AAAA;AAAA,EAGA,mBAAmB,IAA2B;AAC5C,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B,YAAM,mBAAmB,EAAE;AAC3B;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,qBAAqB,GAAG,kBAAkB,GAAG,WAAW;AAChF,UAAM,mBAAmB;AAAA,MACvB,kBAAkB,GAAG;AAAA,MACrB,aAAa,GAAG;AAAA,MAChB,wBAAwB,KAAK,aAAa,MAAM;AAAA,IAClD,CAAC;AAED,SAAK,aAAa,cAAc;AAChC,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,MAAM,yBAAyB,qBAAW;AAAA,EAIxC,YACmB,cACD,aAChB;AACA,UAAM,WAAW;AAHA;AACD;AAAA,EAGlB;AAAA,EARQ,YAAqB;AAAA,EACrB,aAAS,gBAAI;AAAA,EASrB,MAAM,YAAY,MAA6B;AAC7C,UAAM,KAAK,aAAa,QAAQ;AAEhC,QAAI,CAAC,KAAK,aAAa,SAAS;AAE9B,YAAM,KAAK,YAAY,YAAY,IAAI;AACvC;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,QAAI,KAAK,aAAa,MAAM,gBAAgB;AAC1C,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AACA,WAAK,aAAa,cAAc;AAChC,YAAM,KAAK,aAAa,QAAQ;AAAA,IAClC;AACA,SAAK,aAAa,MAAM,SAAS,IAAI;AAAA,EACvC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B,WAAK,YAAY,MAAM;AACvB;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,aAAa,MAAM,aAAa;AAAA,EACvC;AACF;","names":["delay"]}

package/dist/voice/transcription/synchronizer.d.cts

@@ -0,0 +1,86 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { ReadableStream } from 'node:stream/web';
+import type { SentenceTokenizer } from '../../tokenize/index.js';
+import { AudioOutput, type PlaybackFinishedEvent, TextOutput } from '../io.js';
+interface TextSyncOptions {
+    speed: number;
+    hyphenateWord: (word: string) => string[];
+    splitWords: (words: string) => [string, number, number][];
+    sentenceTokenizer: SentenceTokenizer;
+}
+declare class SegmentSynchronizerImpl {
+    private readonly options;
+    private readonly nextInChain;
+    private textData;
+    private audioData;
+    private speed;
+    private outputStream;
+    private outputStreamWriter;
+    private captureTask;
+    private startWallTime?;
+    private startFuture;
+    private closedFuture;
+    private playbackCompleted;
+    private logger;
+    constructor(options: TextSyncOptions, nextInChain: TextOutput);
+    get closed(): boolean;
+    get audioInputEnded(): boolean;
+    get textInputEnded(): boolean;
+    get readable(): ReadableStream<string>;
+    pushAudio(frame: AudioFrame): void;
+    endAudioInput(): void;
+    pushText(text: string): void;
+    endTextInput(): void;
+    markPlaybackFinished(_playbackPosition: number, interrupted: boolean): void;
+    get synchronizedTranscript(): string;
+    private captureTaskImpl;
+    private mainTask;
+    private sleepIfNotClosed;
+    close(): Promise<void>;
+}
+export interface TranscriptionSynchronizerOptions {
+    speed: number;
+    hyphenateWord: (word: string) => string[];
+    splitWords: (words: string) => [string, number, number][];
+    sentenceTokenizer: SentenceTokenizer;
+}
+export declare const defaultTextSyncOptions: TranscriptionSynchronizerOptions;
+export declare class TranscriptionSynchronizer {
+    readonly audioOutput: SyncedAudioOutput;
+    readonly textOutput: SyncedTextOutput;
+    private options;
+    private rotateSegmentTask;
+    private _enabled;
+    private closed;
+    _impl: SegmentSynchronizerImpl;
+    private logger;
+    constructor(nextInChainAudio: AudioOutput, nextInChainText: TextOutput, options?: TranscriptionSynchronizerOptions);
+    get enabled(): boolean;
+    set enabled(enabled: boolean);
+    rotateSegment(): void;
+    close(): Promise<void>;
+    barrier(): Promise<void>;
+    private rotateSegmentTaskImpl;
+}
+declare class SyncedAudioOutput extends AudioOutput {
+    synchronizer: TranscriptionSynchronizer;
+    private nextInChainAudio;
+    private pushedDuration;
+    constructor(synchronizer: TranscriptionSynchronizer, nextInChainAudio: AudioOutput);
+    captureFrame(frame: AudioFrame): Promise<void>;
+    flush(): void;
+    clearBuffer(): void;
+    onPlaybackFinished(ev: PlaybackFinishedEvent): void;
+}
+declare class SyncedTextOutput extends TextOutput {
+    private readonly synchronizer;
+    readonly nextInChain: TextOutput;
+    private capturing;
+    private logger;
+    constructor(synchronizer: TranscriptionSynchronizer, nextInChain: TextOutput);
+    captureText(text: string): Promise<void>;
+    flush(): void;
+}
+export {};
+//# sourceMappingURL=synchronizer.d.ts.map

package/dist/voice/transcription/synchronizer.d.ts

@@ -0,0 +1,86 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { ReadableStream } from 'node:stream/web';
+import type { SentenceTokenizer } from '../../tokenize/index.js';
+import { AudioOutput, type PlaybackFinishedEvent, TextOutput } from '../io.js';
+interface TextSyncOptions {
+    speed: number;
+    hyphenateWord: (word: string) => string[];
+    splitWords: (words: string) => [string, number, number][];
+    sentenceTokenizer: SentenceTokenizer;
+}
+declare class SegmentSynchronizerImpl {
+    private readonly options;
+    private readonly nextInChain;
+    private textData;
+    private audioData;
+    private speed;
+    private outputStream;
+    private outputStreamWriter;
+    private captureTask;
+    private startWallTime?;
+    private startFuture;
+    private closedFuture;
+    private playbackCompleted;
+    private logger;
+    constructor(options: TextSyncOptions, nextInChain: TextOutput);
+    get closed(): boolean;
+    get audioInputEnded(): boolean;
+    get textInputEnded(): boolean;
+    get readable(): ReadableStream<string>;
+    pushAudio(frame: AudioFrame): void;
+    endAudioInput(): void;
+    pushText(text: string): void;
+    endTextInput(): void;
+    markPlaybackFinished(_playbackPosition: number, interrupted: boolean): void;
+    get synchronizedTranscript(): string;
+    private captureTaskImpl;
+    private mainTask;
+    private sleepIfNotClosed;
+    close(): Promise<void>;
+}
+export interface TranscriptionSynchronizerOptions {
+    speed: number;
+    hyphenateWord: (word: string) => string[];
+    splitWords: (words: string) => [string, number, number][];
+    sentenceTokenizer: SentenceTokenizer;
+}
+export declare const defaultTextSyncOptions: TranscriptionSynchronizerOptions;
+export declare class TranscriptionSynchronizer {
+    readonly audioOutput: SyncedAudioOutput;
+    readonly textOutput: SyncedTextOutput;
+    private options;
+    private rotateSegmentTask;
+    private _enabled;
+    private closed;
+    _impl: SegmentSynchronizerImpl;
+    private logger;
+    constructor(nextInChainAudio: AudioOutput, nextInChainText: TextOutput, options?: TranscriptionSynchronizerOptions);
+    get enabled(): boolean;
+    set enabled(enabled: boolean);
+    rotateSegment(): void;
+    close(): Promise<void>;
+    barrier(): Promise<void>;
+    private rotateSegmentTaskImpl;
+}
+declare class SyncedAudioOutput extends AudioOutput {
+    synchronizer: TranscriptionSynchronizer;
+    private nextInChainAudio;
+    private pushedDuration;
+    constructor(synchronizer: TranscriptionSynchronizer, nextInChainAudio: AudioOutput);
+    captureFrame(frame: AudioFrame): Promise<void>;
+    flush(): void;
+    clearBuffer(): void;
+    onPlaybackFinished(ev: PlaybackFinishedEvent): void;
+}
+declare class SyncedTextOutput extends TextOutput {
+    private readonly synchronizer;
+    readonly nextInChain: TextOutput;
+    private capturing;
+    private logger;
+    constructor(synchronizer: TranscriptionSynchronizer, nextInChain: TextOutput);
+    captureText(text: string): Promise<void>;
+    flush(): void;
+}
+export {};
+//# sourceMappingURL=synchronizer.d.ts.map

package/dist/voice/transcription/synchronizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"synchronizer.d.ts","sourceRoot":"","sources":["../../../src/voice/transcription/synchronizer.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAE,cAAc,EAA+B,MAAM,iBAAiB,CAAC;AAGnF,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAGjF,OAAO,EAAE,WAAW,EAAE,KAAK,qBAAqB,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAI/E,UAAU,eAAe;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAC1C,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC1D,iBAAiB,EAAE,iBAAiB,CAAC;CACtC;AAeD,cAAM,uBAAuB;IAgBzB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAhB9B,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,YAAY,CAA4B;IAChD,OAAO,CAAC,kBAAkB,CAAsC;IAChE,OAAO,CAAC,WAAW,CAAgB;IACnC,OAAO,CAAC,aAAa,CAAC,CAAS;IAE/B,OAAO,CAAC,WAAW,CAAwB;IAC3C,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,iBAAiB,CAAkB;IAE3C,OAAO,CAAC,MAAM,CAAS;gBAGJ,OAAO,EAAE,eAAe,EACxB,WAAW,EAAE,UAAU;IA2B1C,IAAI,MAAM,YAET;IAED,IAAI,eAAe,YAElB;IAED,IAAI,cAAc,YAEjB;IAED,IAAI,QAAQ,IAAI,cAAc,CAAC,MAAM,CAAC,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU;IAgB3B,aAAa;IASb,QAAQ,CAAC,IAAI,EAAE,MAAM;IAUrB,YAAY;IAUZ,oBAAoB,CAAC,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO;IAmBpE,IAAI,sBAAsB,IAAI,MAAM,CAKnC;YAEa,eAAe;YAgBf,QAAQ;YAuDR,gBAAgB;IAOxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAS7B;AAED,MAAM,WAAW,gCAAgC;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAC1C,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC1D,iBAAiB,EAAE,iBAAiB,CAAC;CACtC;AAED,eAAO,MAAM,sBAAsB,EAAE,gCAOpC,CAAC;AAEF,qBAAa,yBAAyB;IACpC,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IAEtC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,MAAM,CAAkB;IAGhC,KAAK,EAAE,uBAAuB,CAAC;IAE/B,OAAO,CAAC,MAAM,CAAS;gBAGrB,gBAAgB,EAAE,WAAW,EAC7B,eAAe,EAAE,UAAU,EAC3B,OAAO,GAAE,gCAAyD;IAkBpE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,OAAO,EAO3B;IAED,aAAa;IAaP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAMtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;YAOhB,qBAAqB;CAWpC;AAED,cAAM,iBAAkB,SAAQ,WAAW;IAIhC,YAAY,EAAE,yBAAyB;IAC9C,OAAO,CAAC,gBAAgB;IAJ1B,OAAO,CAAC,cAAc,CAAe;gBAG5B,YAAY,EAAE,yBAAyB,EACtC,gBAAgB,EAAE,WAAW;IAKjC,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBpD,KAAK;IAiBL,WAAW;IAKX,kBAAkB,CAAC,EAAE,EAAE,qBAAqB;CAgB7C;AAED,cAAM,gBAAiB,SAAQ,UAAU;IAKrC,OAAO,CAAC,QAAQ,CAAC,YAAY;aACb,WAAW,EAAE,UAAU;IALzC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,MAAM,CAAS;gBAGJ,YAAY,EAAE,yBAAyB,EACxC,WAAW,EAAE,UAAU;IAKnC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoB9C,KAAK;CAaN"}

package/dist/voice/transcription/synchronizer.js

@@ -0,0 +1,355 @@
+import { delay } from "@std/async";
+import { log } from "../../log.js";
+import { IdentityTransform } from "../../stream/identity_transform.js";
+import { basic } from "../../tokenize/index.js";
+import { Future, Task } from "../../utils.js";
+import { AudioOutput, TextOutput } from "../io.js";
+const STANDARD_SPEECH_RATE = 3.83;
+class SegmentSynchronizerImpl {
+  constructor(options, nextInChain) {
+    this.options = options;
+    this.nextInChain = nextInChain;
+    this.speed = options.speed * STANDARD_SPEECH_RATE;
+    this.textData = {
+      sentenceStream: options.sentenceTokenizer.stream(),
+      pushedText: "",
+      done: false,
+      forwardedHyphens: 0,
+      forwardedText: ""
+    };
+    this.audioData = {
+      pushedDuration: 0,
+      done: false
+    };
+    this.outputStream = new IdentityTransform();
+    this.outputStreamWriter = this.outputStream.writable.getWriter();
+    this.mainTask().then(() => {
+      this.outputStreamWriter.close();
+    }).catch((error) => {
+      this.logger.error({ error }, "mainTask SegmentSynchronizerImpl");
+    });
+    this.captureTask = this.captureTaskImpl();
+  }
+  textData;
+  audioData;
+  speed;
+  outputStream;
+  outputStreamWriter;
+  captureTask;
+  startWallTime;
+  startFuture = new Future();
+  closedFuture = new Future();
+  playbackCompleted = false;
+  logger = log();
+  get closed() {
+    return this.closedFuture.done;
+  }
+  get audioInputEnded() {
+    return this.audioData.done;
+  }
+  get textInputEnded() {
+    return this.textData.done;
+  }
+  get readable() {
+    return this.outputStream.readable;
+  }
+  pushAudio(frame) {
+    if (this.closed) {
+      this.logger.warn("SegmentSynchronizerImpl.pushAudio called after close");
+      return;
+    }
+    const frameDuration = frame.samplesPerChannel / frame.sampleRate;
+    if (!this.startWallTime && frameDuration > 0) {
+      this.startWallTime = Date.now();
+      this.startFuture.resolve();
+    }
+    this.audioData.pushedDuration += frameDuration;
+  }
+  endAudioInput() {
+    if (this.closed) {
+      this.logger.warn("SegmentSynchronizerImpl.endAudioInput called after close");
+      return;
+    }
+    this.audioData.done = true;
+  }
+  pushText(text) {
+    if (this.closed) {
+      this.logger.warn("SegmentSynchronizerImpl.pushText called after close");
+      return;
+    }
+    this.textData.sentenceStream.pushText(text);
+    this.textData.pushedText += text;
+  }
+  endTextInput() {
+    if (this.closed) {
+      this.logger.warn("SegmentSynchronizerImpl.endTextInput called after close");
+      return;
+    }
+    this.textData.done = true;
+    this.textData.sentenceStream.endInput();
+  }
+  markPlaybackFinished(_playbackPosition, interrupted) {
+    if (this.closed) {
+      this.logger.warn("SegmentSynchronizerImpl.markPlaybackFinished called after close");
+      return;
+    }
+    if (!this.textData.done || !this.audioData.done) {
+      this.logger.warn(
+        { textDone: this.textData.done, audioDone: this.audioData.done },
+        "SegmentSynchronizerImpl.markPlaybackFinished called before text/audio input is done"
+      );
+      return;
+    }
+    if (!interrupted) {
+      this.playbackCompleted = true;
+    }
+  }
+  get synchronizedTranscript() {
+    if (this.playbackCompleted) {
+      return this.textData.pushedText;
+    }
+    return this.textData.forwardedText;
+  }
+  async captureTaskImpl() {
+    const reader = this.outputStream.readable.getReader();
+    while (true) {
+      const { done, value: text } = await reader.read();
+      if (done) {
+        break;
+      }
+      this.textData.forwardedText += text;
+      await this.nextInChain.captureText(text);
+    }
+    reader.releaseLock();
+    this.nextInChain.flush();
+  }
+  async mainTask() {
+    await this.startFuture.await;
+    if (this.closed && !this.playbackCompleted) {
+      return;
+    }
+    if (!this.startWallTime) {
+      throw new Error("startWallTime is not set when starting SegmentSynchronizerImpl.mainTask");
+    }
+    for await (const textSegment of this.textData.sentenceStream) {
+      const sentence = textSegment.token;
+      let textCursor = 0;
+      if (this.closed && !this.playbackCompleted) {
+        return;
+      }
+      for (const [word, _, endPos] of this.options.splitWords(sentence)) {
+        if (this.closed && !this.playbackCompleted) {
+          return;
+        }
+        if (this.playbackCompleted) {
+          this.outputStreamWriter.write(sentence.slice(textCursor, endPos));
+          textCursor = endPos;
+          continue;
+        }
+        const wordHphens = this.options.hyphenateWord(word).length;
+        const elapsedSeconds = (Date.now() - this.startWallTime) / 1e3;
+        const targetHyphens = elapsedSeconds * this.options.speed;
+        const hyphensBehind = Math.max(0, targetHyphens - this.textData.forwardedHyphens);
+        let delay2 = Math.max(0, wordHphens - hyphensBehind) / this.speed;
+        if (this.playbackCompleted) {
+          delay2 = 0;
+        }
+        await this.sleepIfNotClosed(delay2 / 2);
+        this.outputStreamWriter.write(sentence.slice(textCursor, endPos));
+        await this.sleepIfNotClosed(delay2 / 2);
+        this.textData.forwardedHyphens += wordHphens;
+        textCursor = endPos;
+      }
+      if (textCursor < sentence.length) {
+        const remaining = sentence.slice(textCursor);
+        this.outputStreamWriter.write(remaining);
+      }
+    }
+  }
+  async sleepIfNotClosed(sleepTimeSeconds) {
+    if (this.closed) {
+      return;
+    }
+    await delay(sleepTimeSeconds * 1e3);
+  }
+  async close() {
+    if (this.closed) {
+      return;
+    }
+    this.closedFuture.resolve();
+    this.startFuture.resolve();
+    this.textData.sentenceStream.close();
+    await this.captureTask;
+  }
+}
+const defaultTextSyncOptions = {
+  speed: 1,
+  hyphenateWord: basic.hyphenateWord,
+  splitWords: basic.splitWords,
+  sentenceTokenizer: new basic.SentenceTokenizer({
+    retainFormat: true
+  })
+};
+class TranscriptionSynchronizer {
+  audioOutput;
+  textOutput;
+  options;
+  rotateSegmentTask;
+  _enabled = true;
+  closed = false;
+  /* @internal */
+  _impl;
+  logger = log();
+  constructor(nextInChainAudio, nextInChainText, options = defaultTextSyncOptions) {
+    this.audioOutput = new SyncedAudioOutput(this, nextInChainAudio);
+    this.textOutput = new SyncedTextOutput(this, nextInChainText);
+    this.options = {
+      speed: options.speed,
+      hyphenateWord: options.hyphenateWord,
+      splitWords: options.splitWords,
+      sentenceTokenizer: options.sentenceTokenizer
+    };
+    this._impl = new SegmentSynchronizerImpl(this.options, nextInChainText);
+    this.rotateSegmentTask = Task.from(
+      (controller) => this.rotateSegmentTaskImpl(controller.signal)
+    );
+  }
+  get enabled() {
+    return this._enabled;
+  }
+  set enabled(enabled) {
+    if (this._enabled === enabled) {
+      return;
+    }
+    this._enabled = enabled;
+    this.rotateSegment();
+  }
+  rotateSegment() {
+    if (this.closed) {
+      return;
+    }
+    if (!this.rotateSegmentTask.done) {
+      this.logger.warn("rotateSegment called while previous segment is still being rotated");
+    }
+    this.rotateSegmentTask = Task.from(
+      (controller) => this.rotateSegmentTaskImpl(controller.signal, this.rotateSegmentTask)
+    );
+  }
+  async close() {
+    this.closed = true;
+    await this.rotateSegmentTask.cancelAndWait();
+    await this._impl.close();
+  }
+  async barrier() {
+    if (this.rotateSegmentTask.done) {
+      return;
+    }
+    await this.rotateSegmentTask.result;
+  }
+  async rotateSegmentTaskImpl(abort, oldTask) {
+    if (oldTask) {
+      await oldTask.result;
+    }
+    if (abort.aborted) {
+      return;
+    }
+    await this._impl.close();
+    this._impl = new SegmentSynchronizerImpl(this.options, this.textOutput.nextInChain);
+  }
+}
+class SyncedAudioOutput extends AudioOutput {
+  constructor(synchronizer, nextInChainAudio) {
+    super(nextInChainAudio.sampleRate, nextInChainAudio);
+    this.synchronizer = synchronizer;
+    this.nextInChainAudio = nextInChainAudio;
+  }
+  pushedDuration = 0;
+  async captureFrame(frame) {
+    await this.synchronizer.barrier();
+    await super.captureFrame(frame);
+    await this.nextInChainAudio.captureFrame(frame);
+    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;
+    if (!this.synchronizer.enabled) {
+      return;
+    }
+    if (this.synchronizer._impl.audioInputEnded) {
+      this.logger.warn(
+        "SegmentSynchronizerImpl audio marked as ended in capture audio, rotating segment"
+      );
+      this.synchronizer.rotateSegment();
+      await this.synchronizer.barrier();
+    }
+    this.synchronizer._impl.pushAudio(frame);
+  }
+  flush() {
+    super.flush();
+    this.nextInChainAudio.flush();
+    if (!this.synchronizer.enabled) {
+      return;
+    }
+    if (!this.pushedDuration) {
+      this.synchronizer.rotateSegment();
+      return;
+    }
+    this.synchronizer._impl.endAudioInput();
+  }
+  clearBuffer() {
+    this.nextInChainAudio.clearBuffer();
+  }
+  // this is going to be automatically called by the next_in_chain
+  onPlaybackFinished(ev) {
+    if (!this.synchronizer.enabled) {
+      super.onPlaybackFinished(ev);
+      return;
+    }
+    this.synchronizer._impl.markPlaybackFinished(ev.playbackPosition, ev.interrupted);
+    super.onPlaybackFinished({
+      playbackPosition: ev.playbackPosition,
+      interrupted: ev.interrupted,
+      synchronizedTranscript: this.synchronizer._impl.synchronizedTranscript
+    });
+    this.synchronizer.rotateSegment();
+    this.pushedDuration = 0;
+  }
+}
+class SyncedTextOutput extends TextOutput {
+  constructor(synchronizer, nextInChain) {
+    super(nextInChain);
+    this.synchronizer = synchronizer;
+    this.nextInChain = nextInChain;
+  }
+  capturing = false;
+  logger = log();
+  async captureText(text) {
+    await this.synchronizer.barrier();
+    if (!this.synchronizer.enabled) {
+      await this.nextInChain.captureText(text);
+      return;
+    }
+    this.capturing = true;
+    if (this.synchronizer._impl.textInputEnded) {
+      this.logger.warn(
+        "SegmentSynchronizerImpl text marked as ended in capture text, rotating segment"
+      );
+      this.synchronizer.rotateSegment();
+      await this.synchronizer.barrier();
+    }
+    this.synchronizer._impl.pushText(text);
+  }
+  flush() {
+    if (!this.synchronizer.enabled) {
+      this.nextInChain.flush();
+      return;
+    }
+    if (!this.capturing) {
+      return;
+    }
+    this.capturing = false;
+    this.synchronizer._impl.endTextInput();
+  }
+}
+export {
+  TranscriptionSynchronizer,
+  defaultTextSyncOptions
+};
+//# sourceMappingURL=synchronizer.js.map

package/dist/voice/transcription/synchronizer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/voice/transcription/synchronizer.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { delay } from '@std/async';\nimport type { ReadableStream, WritableStreamDefaultWriter } from 'node:stream/web';\nimport { log } from '../../log.js';\nimport { IdentityTransform } from '../../stream/identity_transform.js';\nimport type { SentenceStream, SentenceTokenizer } from '../../tokenize/index.js';\nimport { basic } from '../../tokenize/index.js';\nimport { Future, Task } from '../../utils.js';\nimport { AudioOutput, type PlaybackFinishedEvent, TextOutput } from '../io.js';\n\nconst STANDARD_SPEECH_RATE = 3.83; // hyphens (syllables) per second\n\ninterface TextSyncOptions {\n  speed: number;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n  sentenceTokenizer: SentenceTokenizer;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedText: string;\n}\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\nclass SegmentSynchronizerImpl {\n  private textData: TextData;\n  private audioData: AudioData;\n  private speed: number;\n  private outputStream: IdentityTransform<string>;\n  private outputStreamWriter: WritableStreamDefaultWriter<string>;\n  private captureTask: Promise<void>;\n  private startWallTime?: number;\n\n  private startFuture: Future = new Future();\n  private closedFuture: Future = new Future();\n  private playbackCompleted: boolean = false;\n\n  private logger = log();\n\n  constructor(\n    private readonly options: TextSyncOptions,\n    private readonly nextInChain: TextOutput,\n  ) {\n    this.speed = options.speed * STANDARD_SPEECH_RATE; // hyphens per second\n    this.textData = {\n      sentenceStream: options.sentenceTokenizer.stream(),\n      pushedText: '',\n      done: false,\n      forwardedHyphens: 0,\n      forwardedText: '',\n    };\n    this.audioData = {\n      pushedDuration: 0,\n      done: false,\n    };\n    this.outputStream = new IdentityTransform();\n    this.outputStreamWriter = this.outputStream.writable.getWriter();\n\n    this.mainTask()\n      .then(() => {\n        this.outputStreamWriter.close();\n      })\n      .catch((error) => {\n        this.logger.error({ error }, 'mainTask SegmentSynchronizerImpl');\n      });\n    this.captureTask = this.captureTaskImpl();\n  }\n\n  get closed() {\n    return this.closedFuture.done;\n  }\n\n  get audioInputEnded() {\n    return this.audioData.done;\n  }\n\n  get textInputEnded() {\n    return this.textData.done;\n  }\n\n  get readable(): ReadableStream<string> {\n    return this.outputStream.readable;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.pushAudio called after close');\n      return;\n    }\n    // TODO(AJS-102): use frame.durationMs once available in rtc-node\n    const frameDuration = frame.samplesPerChannel / frame.sampleRate;\n\n    if (!this.startWallTime && frameDuration > 0) {\n      this.startWallTime = Date.now();\n      this.startFuture.resolve();\n    }\n\n    this.audioData.pushedDuration += frameDuration;\n  }\n\n  endAudioInput() {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.endAudioInput called after close');\n      return;\n    }\n\n    this.audioData.done = true;\n  }\n\n  pushText(text: string) {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.pushText called after close');\n      return;\n    }\n\n    this.textData.sentenceStream.pushText(text);\n    this.textData.pushedText += text;\n  }\n\n  endTextInput() {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.endTextInput called after close');\n      return;\n    }\n\n    this.textData.done = true;\n    this.textData.sentenceStream.endInput();\n  }\n\n  markPlaybackFinished(_playbackPosition: number, interrupted: boolean) {\n    if (this.closed) {\n      this.logger.warn('SegmentSynchronizerImpl.markPlaybackFinished called after close');\n      return;\n    }\n\n    if (!this.textData.done || !this.audioData.done) {\n      this.logger.warn(\n        { textDone: this.textData.done, audioDone: this.audioData.done },\n        'SegmentSynchronizerImpl.markPlaybackFinished called before text/audio input is done',\n      );\n      return;\n    }\n\n    if (!interrupted) {\n      this.playbackCompleted = true;\n    }\n  }\n\n  get synchronizedTranscript(): string {\n    if (this.playbackCompleted) {\n      return this.textData.pushedText;\n    }\n    return this.textData.forwardedText;\n  }\n\n  private async captureTaskImpl() {\n    // Don't use a for-await loop here, because exiting the loop will close the writer in the\n    // outputStream, which will cause an error in the mainTask.then method.\n    const reader = this.outputStream.readable.getReader();\n    while (true) {\n      const { done, value: text } = await reader.read();\n      if (done) {\n        break;\n      }\n      this.textData.forwardedText += text;\n      await this.nextInChain.captureText(text);\n    }\n    reader.releaseLock();\n    this.nextInChain.flush();\n  }\n\n  private async mainTask(): Promise<void> {\n    await this.startFuture.await;\n\n    if (this.closed && !this.playbackCompleted) {\n      return;\n    }\n\n    if (!this.startWallTime) {\n      throw new Error('startWallTime is not set when starting SegmentSynchronizerImpl.mainTask');\n    }\n\n    for await (const textSegment of this.textData.sentenceStream) {\n      const sentence = textSegment.token;\n\n      let textCursor = 0;\n      if (this.closed && !this.playbackCompleted) {\n        return;\n      }\n\n      for (const [word, _, endPos] of this.options.splitWords(sentence)) {\n        if (this.closed && !this.playbackCompleted) {\n          return;\n        }\n\n        if (this.playbackCompleted) {\n          this.outputStreamWriter.write(sentence.slice(textCursor, endPos));\n          textCursor = endPos;\n          continue;\n        }\n\n        const wordHphens = this.options.hyphenateWord(word).length;\n        const elapsedSeconds = (Date.now() - this.startWallTime) / 1000;\n        const targetHyphens = elapsedSeconds * this.options.speed;\n        const hyphensBehind = Math.max(0, targetHyphens - this.textData.forwardedHyphens);\n        let delay = Math.max(0, wordHphens - hyphensBehind) / this.speed;\n\n        if (this.playbackCompleted) {\n          delay = 0;\n        }\n\n        await this.sleepIfNotClosed(delay / 2);\n        this.outputStreamWriter.write(sentence.slice(textCursor, endPos));\n        await this.sleepIfNotClosed(delay / 2);\n\n        this.textData.forwardedHyphens += wordHphens;\n        textCursor = endPos;\n      }\n\n      if (textCursor < sentence.length) {\n        const remaining = sentence.slice(textCursor);\n        this.outputStreamWriter.write(remaining);\n      }\n    }\n  }\n\n  private async sleepIfNotClosed(sleepTimeSeconds: number) {\n    if (this.closed) {\n      return;\n    }\n    await delay(sleepTimeSeconds * 1000);\n  }\n\n  async close(): Promise<void> {\n    if (this.closed) {\n      return;\n    }\n    this.closedFuture.resolve();\n    this.startFuture.resolve(); // avoid deadlock of mainTaskImpl in case it never started\n    this.textData.sentenceStream.close();\n    await this.captureTask;\n  }\n}\n\nexport interface TranscriptionSynchronizerOptions {\n  speed: number;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n  sentenceTokenizer: SentenceTokenizer;\n}\n\nexport const defaultTextSyncOptions: TranscriptionSynchronizerOptions = {\n  speed: 1,\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n  sentenceTokenizer: new basic.SentenceTokenizer({\n    retainFormat: true,\n  }),\n};\n\nexport class TranscriptionSynchronizer {\n  readonly audioOutput: SyncedAudioOutput;\n  readonly textOutput: SyncedTextOutput;\n\n  private options: TextSyncOptions;\n  private rotateSegmentTask: Task<void>;\n  private _enabled: boolean = true;\n  private closed: boolean = false;\n\n  /* @internal */\n  _impl: SegmentSynchronizerImpl;\n\n  private logger = log();\n\n  constructor(\n    nextInChainAudio: AudioOutput,\n    nextInChainText: TextOutput,\n    options: TranscriptionSynchronizerOptions = defaultTextSyncOptions,\n  ) {\n    this.audioOutput = new SyncedAudioOutput(this, nextInChainAudio);\n    this.textOutput = new SyncedTextOutput(this, nextInChainText);\n    this.options = {\n      speed: options.speed,\n      hyphenateWord: options.hyphenateWord,\n      splitWords: options.splitWords,\n      sentenceTokenizer: options.sentenceTokenizer,\n    };\n\n    // initial segment/first segment, recreated for each new segment\n    this._impl = new SegmentSynchronizerImpl(this.options, nextInChainText);\n    this.rotateSegmentTask = Task.from((controller) =>\n      this.rotateSegmentTaskImpl(controller.signal),\n    );\n  }\n\n  get enabled(): boolean {\n    return this._enabled;\n  }\n\n  set enabled(enabled: boolean) {\n    if (this._enabled === enabled) {\n      return;\n    }\n\n    this._enabled = enabled;\n    this.rotateSegment();\n  }\n\n  rotateSegment() {\n    if (this.closed) {\n      return;\n    }\n\n    if (!this.rotateSegmentTask.done) {\n      this.logger.warn('rotateSegment called while previous segment is still being rotated');\n    }\n    this.rotateSegmentTask = Task.from((controller) =>\n      this.rotateSegmentTaskImpl(controller.signal, this.rotateSegmentTask),\n    );\n  }\n\n  async close(): Promise<void> {\n    this.closed = true;\n    await this.rotateSegmentTask.cancelAndWait();\n    await this._impl.close();\n  }\n\n  async barrier(): Promise<void> {\n    if (this.rotateSegmentTask.done) {\n      return;\n    }\n    await this.rotateSegmentTask.result;\n  }\n\n  private async rotateSegmentTaskImpl(abort: AbortSignal, oldTask?: Task<void>) {\n    if (oldTask) {\n      await oldTask.result;\n    }\n\n    if (abort.aborted) {\n      return;\n    }\n    await this._impl.close();\n    this._impl = new SegmentSynchronizerImpl(this.options, this.textOutput.nextInChain);\n  }\n}\n\nclass SyncedAudioOutput extends AudioOutput {\n  private pushedDuration: number = 0.0;\n\n  constructor(\n    public synchronizer: TranscriptionSynchronizer,\n    private nextInChainAudio: AudioOutput,\n  ) {\n    super(nextInChainAudio.sampleRate, nextInChainAudio);\n  }\n\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    // using barrier() on capture should be sufficient, flush() must not be called if\n    // capture_frame isn't completed\n    await this.synchronizer.barrier();\n\n    await super.captureFrame(frame);\n    await this.nextInChainAudio.captureFrame(frame); // passthrough audio\n\n    // TODO(AJS-102): use frame.durationMs once available in rtc-node\n    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n\n    if (!this.synchronizer.enabled) {\n      return;\n    }\n\n    if (this.synchronizer._impl.audioInputEnded) {\n      this.logger.warn(\n        'SegmentSynchronizerImpl audio marked as ended in capture audio, rotating segment',\n      );\n      this.synchronizer.rotateSegment();\n      await this.synchronizer.barrier();\n    }\n    this.synchronizer._impl.pushAudio(frame);\n  }\n\n  flush() {\n    super.flush();\n    this.nextInChainAudio.flush();\n\n    if (!this.synchronizer.enabled) {\n      return;\n    }\n\n    if (!this.pushedDuration) {\n      // in case there is no audio after the text was pushed, rotate the segment\n      this.synchronizer.rotateSegment();\n      return;\n    }\n\n    this.synchronizer._impl.endAudioInput();\n  }\n\n  clearBuffer() {\n    this.nextInChainAudio.clearBuffer();\n  }\n\n  // this is going to be automatically called by the next_in_chain\n  onPlaybackFinished(ev: PlaybackFinishedEvent) {\n    if (!this.synchronizer.enabled) {\n      super.onPlaybackFinished(ev);\n      return;\n    }\n\n    this.synchronizer._impl.markPlaybackFinished(ev.playbackPosition, ev.interrupted);\n    super.onPlaybackFinished({\n      playbackPosition: ev.playbackPosition,\n      interrupted: ev.interrupted,\n      synchronizedTranscript: this.synchronizer._impl.synchronizedTranscript,\n    });\n\n    this.synchronizer.rotateSegment();\n    this.pushedDuration = 0.0;\n  }\n}\n\nclass SyncedTextOutput extends TextOutput {\n  private capturing: boolean = false;\n  private logger = log();\n\n  constructor(\n    private readonly synchronizer: TranscriptionSynchronizer,\n    public readonly nextInChain: TextOutput,\n  ) {\n    super(nextInChain);\n  }\n\n  async captureText(text: string): Promise<void> {\n    await this.synchronizer.barrier();\n\n    if (!this.synchronizer.enabled) {\n      // pass through to the next in chain\n      await this.nextInChain.captureText(text);\n      return;\n    }\n\n    this.capturing = true;\n    if (this.synchronizer._impl.textInputEnded) {\n      this.logger.warn(\n        'SegmentSynchronizerImpl text marked as ended in capture text, rotating segment',\n      );\n      this.synchronizer.rotateSegment();\n      await this.synchronizer.barrier();\n    }\n    this.synchronizer._impl.pushText(text);\n  }\n\n  flush() {\n    if (!this.synchronizer.enabled) {\n      this.nextInChain.flush(); // passthrough text if the synchronizer is disabled\n      return;\n    }\n\n    if (!this.capturing) {\n      return;\n    }\n\n    this.capturing = false;\n    this.synchronizer._impl.endTextInput();\n  }\n}\n"],"mappings":"AAIA,SAAS,aAAa;AAEtB,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAElC,SAAS,aAAa;AACtB,SAAS,QAAQ,YAAY;AAC7B,SAAS,aAAyC,kBAAkB;AAEpE,MAAM,uBAAuB;AAsB7B,MAAM,wBAAwB;AAAA,EAe5B,YACmB,SACA,aACjB;AAFiB;AACA;AAEjB,SAAK,QAAQ,QAAQ,QAAQ;AAC7B,SAAK,WAAW;AAAA,MACd,gBAAgB,QAAQ,kBAAkB,OAAO;AAAA,MACjD,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,kBAAkB;AAAA,MAClB,eAAe;AAAA,IACjB;AACA,SAAK,YAAY;AAAA,MACf,gBAAgB;AAAA,MAChB,MAAM;AAAA,IACR;AACA,SAAK,eAAe,IAAI,kBAAkB;AAC1C,SAAK,qBAAqB,KAAK,aAAa,SAAS,UAAU;AAE/D,SAAK,SAAS,EACX,KAAK,MAAM;AACV,WAAK,mBAAmB,MAAM;AAAA,IAChC,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,WAAK,OAAO,MAAM,EAAE,MAAM,GAAG,kCAAkC;AAAA,IACjE,CAAC;AACH,SAAK,cAAc,KAAK,gBAAgB;AAAA,EAC1C;AAAA,EAzCQ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,cAAsB,IAAI,OAAO;AAAA,EACjC,eAAuB,IAAI,OAAO;AAAA,EAClC,oBAA6B;AAAA,EAE7B,SAAS,IAAI;AAAA,EA+BrB,IAAI,SAAS;AACX,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,IAAI,kBAAkB;AACpB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA,EAEA,IAAI,iBAAiB;AACnB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,IAAI,WAAmC;AACrC,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,UAAU,OAAmB;AAC3B,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,sDAAsD;AACvE;AAAA,IACF;AAEA,UAAM,gBAAgB,MAAM,oBAAoB,MAAM;AAEtD,QAAI,CAAC,KAAK,iBAAiB,gBAAgB,GAAG;AAC5C,WAAK,gBAAgB,KAAK,IAAI;AAC9B,WAAK,YAAY,QAAQ;AAAA,IAC3B;AAEA,SAAK,UAAU,kBAAkB;AAAA,EACnC;AAAA,EAEA,gBAAgB;AACd,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,0DAA0D;AAC3E;AAAA,IACF;AAEA,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA,EAEA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,qDAAqD;AACtE;AAAA,IACF;AAEA,SAAK,SAAS,eAAe,SAAS,IAAI;AAC1C,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,eAAe;AACb,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,yDAAyD;AAC1E;AAAA,IACF;AAEA,SAAK,SAAS,OAAO;AACrB,SAAK,SAAS,eAAe,SAAS;AAAA,EACxC;AAAA,EAEA,qBAAqB,mBAA2B,aAAsB;AACpE,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,KAAK,iEAAiE;AAClF;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,SAAS,QAAQ,CAAC,KAAK,UAAU,MAAM;AAC/C,WAAK,OAAO;AAAA,QACV,EAAE,UAAU,KAAK,SAAS,MAAM,WAAW,KAAK,UAAU,KAAK;AAAA,QAC/D;AAAA,MACF;AACA;AAAA,IACF;AAEA,QAAI,CAAC,aAAa;AAChB,WAAK,oBAAoB;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,IAAI,yBAAiC;AACnC,QAAI,KAAK,mBAAmB;AAC1B,aAAO,KAAK,SAAS;AAAA,IACvB;AACA,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,MAAc,kBAAkB;AAG9B,UAAM,SAAS,KAAK,aAAa,SAAS,UAAU;AACpD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,OAAO,KAAK,IAAI,MAAM,OAAO,KAAK;AAChD,UAAI,MAAM;AACR;AAAA,MACF;AACA,WAAK,SAAS,iBAAiB;AAC/B,YAAM,KAAK,YAAY,YAAY,IAAI;AAAA,IACzC;AACA,WAAO,YAAY;AACnB,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA,EAEA,MAAc,WAA0B;AACtC,UAAM,KAAK,YAAY;AAEvB,QAAI,KAAK,UAAU,CAAC,KAAK,mBAAmB;AAC1C;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,eAAe;AACvB,YAAM,IAAI,MAAM,yEAAyE;AAAA,IAC3F;AAEA,qBAAiB,eAAe,KAAK,SAAS,gBAAgB;AAC5D,YAAM,WAAW,YAAY;AAE7B,UAAI,aAAa;AACjB,UAAI,KAAK,UAAU,CAAC,KAAK,mBAAmB;AAC1C;AAAA,MACF;AAEA,iBAAW,CAAC,MAAM,GAAG,MAAM,KAAK,KAAK,QAAQ,WAAW,QAAQ,GAAG;AACjE,YAAI,KAAK,UAAU,CAAC,KAAK,mBAAmB;AAC1C;AAAA,QACF;AAEA,YAAI,KAAK,mBAAmB;AAC1B,eAAK,mBAAmB,MAAM,SAAS,MAAM,YAAY,MAAM,CAAC;AAChE,uBAAa;AACb;AAAA,QACF;AAEA,cAAM,aAAa,KAAK,QAAQ,cAAc,IAAI,EAAE;AACpD,cAAM,kBAAkB,KAAK,IAAI,IAAI,KAAK,iBAAiB;AAC3D,cAAM,gBAAgB,iBAAiB,KAAK,QAAQ;AACpD,cAAM,gBAAgB,KAAK,IAAI,GAAG,gBAAgB,KAAK,SAAS,gBAAgB;AAChF,YAAIA,SAAQ,KAAK,IAAI,GAAG,aAAa,aAAa,IAAI,KAAK;AAE3D,YAAI,KAAK,mBAAmB;AAC1B,UAAAA,SAAQ;AAAA,QACV;AAEA,cAAM,KAAK,iBAAiBA,SAAQ,CAAC;AACrC,aAAK,mBAAmB,MAAM,SAAS,MAAM,YAAY,MAAM,CAAC;AAChE,cAAM,KAAK,iBAAiBA,SAAQ,CAAC;AAErC,aAAK,SAAS,oBAAoB;AAClC,qBAAa;AAAA,MACf;AAEA,UAAI,aAAa,SAAS,QAAQ;AAChC,cAAM,YAAY,SAAS,MAAM,UAAU;AAC3C,aAAK,mBAAmB,MAAM,SAAS;AAAA,MACzC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBAAiB,kBAA0B;AACvD,QAAI,KAAK,QAAQ;AACf;AAAA,IACF;AACA,UAAM,MAAM,mBAAmB,GAAI;AAAA,EACrC;AAAA,EAEA,MAAM,QAAuB;AAC3B,QAAI,KAAK,QAAQ;AACf;AAAA,IACF;AACA,SAAK,aAAa,QAAQ;AAC1B,SAAK,YAAY,QAAQ;AACzB,SAAK,SAAS,eAAe,MAAM;AACnC,UAAM,KAAK;AAAA,EACb;AACF;AASO,MAAM,yBAA2D;AAAA,EACtE,OAAO;AAAA,EACP,eAAe,MAAM;AAAA,EACrB,YAAY,MAAM;AAAA,EAClB,mBAAmB,IAAI,MAAM,kBAAkB;AAAA,IAC7C,cAAc;AAAA,EAChB,CAAC;AACH;AAEO,MAAM,0BAA0B;AAAA,EAC5B;AAAA,EACA;AAAA,EAED;AAAA,EACA;AAAA,EACA,WAAoB;AAAA,EACpB,SAAkB;AAAA;AAAA,EAG1B;AAAA,EAEQ,SAAS,IAAI;AAAA,EAErB,YACE,kBACA,iBACA,UAA4C,wBAC5C;AACA,SAAK,cAAc,IAAI,kBAAkB,MAAM,gBAAgB;AAC/D,SAAK,aAAa,IAAI,iBAAiB,MAAM,eAAe;AAC5D,SAAK,UAAU;AAAA,MACb,OAAO,QAAQ;AAAA,MACf,eAAe,QAAQ;AAAA,MACvB,YAAY,QAAQ;AAAA,MACpB,mBAAmB,QAAQ;AAAA,IAC7B;AAGA,SAAK,QAAQ,IAAI,wBAAwB,KAAK,SAAS,eAAe;AACtE,SAAK,oBAAoB,KAAK;AAAA,MAAK,CAAC,eAClC,KAAK,sBAAsB,WAAW,MAAM;AAAA,IAC9C;AAAA,EACF;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAAQ,SAAkB;AAC5B,QAAI,KAAK,aAAa,SAAS;AAC7B;AAAA,IACF;AAEA,SAAK,WAAW;AAChB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,gBAAgB;AACd,QAAI,KAAK,QAAQ;AACf;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,kBAAkB,MAAM;AAChC,WAAK,OAAO,KAAK,oEAAoE;AAAA,IACvF;AACA,SAAK,oBAAoB,KAAK;AAAA,MAAK,CAAC,eAClC,KAAK,sBAAsB,WAAW,QAAQ,KAAK,iBAAiB;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,MAAM,QAAuB;AAC3B,SAAK,SAAS;AACd,UAAM,KAAK,kBAAkB,cAAc;AAC3C,UAAM,KAAK,MAAM,MAAM;AAAA,EACzB;AAAA,EAEA,MAAM,UAAyB;AAC7B,QAAI,KAAK,kBAAkB,MAAM;AAC/B;AAAA,IACF;AACA,UAAM,KAAK,kBAAkB;AAAA,EAC/B;AAAA,EAEA,MAAc,sBAAsB,OAAoB,SAAsB;AAC5E,QAAI,SAAS;AACX,YAAM,QAAQ;AAAA,IAChB;AAEA,QAAI,MAAM,SAAS;AACjB;AAAA,IACF;AACA,UAAM,KAAK,MAAM,MAAM;AACvB,SAAK,QAAQ,IAAI,wBAAwB,KAAK,SAAS,KAAK,WAAW,WAAW;AAAA,EACpF;AACF;AAEA,MAAM,0BAA0B,YAAY;AAAA,EAG1C,YACS,cACC,kBACR;AACA,UAAM,iBAAiB,YAAY,gBAAgB;AAH5C;AACC;AAAA,EAGV;AAAA,EAPQ,iBAAyB;AAAA,EASjC,MAAM,aAAa,OAAkC;AAGnD,UAAM,KAAK,aAAa,QAAQ;AAEhC,UAAM,MAAM,aAAa,KAAK;AAC9B,UAAM,KAAK,iBAAiB,aAAa,KAAK;AAG9C,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AAEvD,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,MAAM,iBAAiB;AAC3C,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AACA,WAAK,aAAa,cAAc;AAChC,YAAM,KAAK,aAAa,QAAQ;AAAA,IAClC;AACA,SAAK,aAAa,MAAM,UAAU,KAAK;AAAA,EACzC;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,iBAAiB,MAAM;AAE5B,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,gBAAgB;AAExB,WAAK,aAAa,cAAc;AAChC;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,cAAc;AAAA,EACxC;AAAA,EAEA,cAAc;AACZ,SAAK,iBAAiB,YAAY;AAAA,EACpC;AAAA;AAAA,EAGA,mBAAmB,IAA2B;AAC5C,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B,YAAM,mBAAmB,EAAE;AAC3B;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,qBAAqB,GAAG,kBAAkB,GAAG,WAAW;AAChF,UAAM,mBAAmB;AAAA,MACvB,kBAAkB,GAAG;AAAA,MACrB,aAAa,GAAG;AAAA,MAChB,wBAAwB,KAAK,aAAa,MAAM;AAAA,IAClD,CAAC;AAED,SAAK,aAAa,cAAc;AAChC,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,MAAM,yBAAyB,WAAW;AAAA,EAIxC,YACmB,cACD,aAChB;AACA,UAAM,WAAW;AAHA;AACD;AAAA,EAGlB;AAAA,EARQ,YAAqB;AAAA,EACrB,SAAS,IAAI;AAAA,EASrB,MAAM,YAAY,MAA6B;AAC7C,UAAM,KAAK,aAAa,QAAQ;AAEhC,QAAI,CAAC,KAAK,aAAa,SAAS;AAE9B,YAAM,KAAK,YAAY,YAAY,IAAI;AACvC;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,QAAI,KAAK,aAAa,MAAM,gBAAgB;AAC1C,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AACA,WAAK,aAAa,cAAc;AAChC,YAAM,KAAK,aAAa,QAAQ;AAAA,IAClC;AACA,SAAK,aAAa,MAAM,SAAS,IAAI;AAAA,EACvC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,aAAa,SAAS;AAC9B,WAAK,YAAY,MAAM;AACvB;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,aAAa,MAAM,aAAa;AAAA,EACvC;AACF;","names":["delay"]}