@livekit/agents 1.1.0 → 1.2.0

package/dist/transcription.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/transcription.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TranscriptionSegment } from '@livekit/protocol';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { basic } from './tokenize/index.js';\nimport type { SentenceStream, SentenceTokenizer } from './tokenize/tokenizer.js';\nimport { AsyncIterableQueue, Future, shortuuid } from './utils.js';\n\n// standard speech rate in hyphens/ms\nconst STANDARD_SPEECH_RATE = 3830;\n\nexport interface TextSyncOptions {\n  language: string;\n  speed: number;\n  newSentenceDelay: number;\n  sentenceTokenizer: SentenceTokenizer;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n}\n\nexport const defaultTextSyncOptions: TextSyncOptions = {\n  language: '',\n  speed: 1,\n  newSentenceDelay: 400,\n  sentenceTokenizer: new basic.SentenceTokenizer(),\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n};\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedSentences: number;\n}\n\ntype SyncCallbacks = {\n  textUpdated: (text: TranscriptionSegment) => void;\n};\n\nexport class TextAudioSynchronizer extends (EventEmitter as new () => TypedEmitter<SyncCallbacks>) {\n  #opts: TextSyncOptions;\n  #speed: number;\n\n  #closed = false;\n  #interrupted = false;\n  #closeFut = new Future();\n\n  #playingSegIndex = -1;\n  #finishedSegIndex = -1;\n\n  #textQChanged = new AsyncIterableQueue<number>();\n  #textQ: (TextData | undefined)[] = [];\n  #audioQChanged = new AsyncIterableQueue<number>();\n  #audioQ: (AudioData | undefined)[] = [];\n\n  #playedText = '';\n  #task?: Promise<void>;\n\n  #audioData?: AudioData;\n  #textData?: TextData;\n\n  constructor(opts: TextSyncOptions) {\n    super();\n\n    this.#opts = opts;\n    this.#speed = opts.speed * STANDARD_SPEECH_RATE;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    this.#checkNotClosed();\n    if (!this.#audioData) {\n      this.#audioData = { pushedDuration: 0, done: false };\n      this.#audioQ.push(this.#audioData);\n      this.#audioQChanged.put(1);\n    }\n    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  pushText(text: string) {\n    this.#checkNotClosed();\n    if (!this.#textData) {\n      this.#textData = {\n        sentenceStream: this.#opts.sentenceTokenizer.stream(),\n        pushedText: '',\n        done: false,\n        forwardedHyphens: 0,\n        forwardedSentences: 0,\n      };\n      this.#textQ.push(this.#textData);\n      this.#textQChanged.put(1);\n    }\n\n    this.#textData.pushedText += text;\n    this.#textData.sentenceStream.pushText(text);\n  }\n\n  markAudioSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#audioData) {\n      // create empty audio data if none exists\n      this.pushAudio(new AudioFrame(new Int16Array(), 24000, 1, 0));\n    }\n\n    this.#audioData!.done = true;\n    this.#audioData = undefined;\n  }\n\n  markTextSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#textData) {\n      this.pushText('');\n    }\n\n    this.#textData!.done = true;\n    this.#textData?.sentenceStream.flush();\n    this.#textData?.sentenceStream.close();\n    this.#textData = undefined;\n  }\n\n  segmentPlayoutStarted() {\n    this.#checkNotClosed();\n    this.#playingSegIndex++;\n\n    if (!this.#task) {\n      this.#task = this.#mainLoop();\n    }\n  }\n\n  segmentPlayoutFinished() {\n    this.#checkNotClosed();\n    this.#finishedSegIndex++;\n  }\n\n  get playedText(): string {\n    return this.#playedText;\n  }\n\n  async close(interrupt: boolean) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#interrupted = interrupt;\n    this.#closeFut.resolve();\n\n    for (const textData of this.#textQ) {\n      textData?.sentenceStream.close();\n    }\n\n    this.#textQ.push(undefined);\n    this.#audioQ.push(undefined);\n    this.#textQChanged.put(1);\n    this.#audioQChanged.put(1);\n\n    await this.#task;\n  }\n\n  async #mainLoop() {\n    let segIndex = 0;\n    let qDone = false;\n\n    while (!qDone) {\n      await this.#textQChanged.next();\n      await this.#audioQChanged.next();\n\n      while (this.#textQ.length && this.#audioQ.length) {\n        const textData = this.#textQ.pop();\n        const audioData = this.#audioQ.pop();\n\n        if (!(textData && audioData)) {\n          qDone = true;\n          break;\n        }\n\n        // wait for segment to start playing\n        while (!this.#closed) {\n          if (this.#playingSegIndex >= segIndex) break;\n          await this.#sleepIfNotClosed(125);\n        }\n\n        const sentenceStream = textData.sentenceStream;\n        const forwardStartTime = Date.now();\n\n        for await (const ev of sentenceStream) {\n          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);\n        }\n\n        segIndex++;\n      }\n    }\n  }\n\n  async #syncSentence(\n    segIndex: number,\n    segStartTime: number,\n    textData: TextData,\n    audioData: AudioData,\n    sentence: string,\n  ) {\n    let realSpeed: number | undefined;\n    if (audioData.pushedDuration > 0 && audioData.done) {\n      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;\n    }\n\n    const segId = shortuuid('SG_');\n    const words = this.#opts.splitWords(sentence);\n    const processedWords: string[] = [];\n\n    const ogText = this.#playedText;\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    for (const [word, _, end] of words) {\n      if (segIndex <= this.#finishedSegIndex) break;\n      if (this.#interrupted) return;\n\n      const wordHyphens = this.#opts.hyphenateWord(word).length;\n      processedWords.push(word);\n\n      const elapsed = Date.now() - segStartTime;\n      const text = sentence.slice(0, end); // TODO: rstrip punctuations\n\n      let speed = this.#speed;\n      let delay: number;\n      if (realSpeed) {\n        speed = realSpeed;\n        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;\n        const hyphPauses = estimatedPausesMs * speed;\n        const targetHyphens = Math.round(speed * elapsed);\n        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;\n        const toWaitHyphens = Math.max(0, wordHyphens - dt);\n        delay = toWaitHyphens / speed;\n      } else {\n        delay = wordHyphens / speed;\n      }\n\n      const firstDelay = Math.min(delay / 2, 2 / speed);\n      await this.#sleepIfNotClosed(firstDelay * 1000000);\n\n      this.emit(\n        'textUpdated',\n        new TranscriptionSegment({\n          id: segId,\n          text: text,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          final: false,\n          language: this.#opts.language,\n        }),\n      );\n\n      this.#playedText = `${ogText} ${text}`;\n      await this.#sleepIfNotClosed((delay - firstDelay) * 1000000);\n      textData.forwardedHyphens += wordHyphens;\n    }\n\n    this.emit(\n      'textUpdated',\n      new TranscriptionSegment({\n        id: segId,\n        text: sentence,\n        startTime: BigInt(0),\n        endTime: BigInt(0),\n        final: true,\n        language: this.#opts.language,\n      }),\n    );\n\n    this.#playedText = `${ogText} ${sentence}`;\n\n    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);\n    textData.forwardedSentences++;\n  }\n\n  async #sleepIfNotClosed(delay: number) {\n    await Promise.race([\n      this.#closeFut.await,\n      new Promise((resolve) => setTimeout(resolve, delay)),\n    ]);\n  }\n\n  #calcHyphens(text: string): string[] {\n    const hyphens: string[] = [];\n    const words = this.#opts.splitWords(text);\n    for (const word of words) {\n      const n = this.#opts.hyphenateWord(word[0]);\n      hyphens.push(...n);\n    }\n    return hyphens;\n  }\n\n  #checkNotClosed() {\n    if (this.#closed) {\n      throw new Error('TextAudioSynchronizer is closed');\n    }\n  }\n}\n"],"mappings":"AAGA,SAAS,4BAA4B;AACrC,SAAS,kBAAkB;AAE3B,SAAS,oBAAoB;AAC7B,SAAS,aAAa;AAEtB,SAAS,oBAAoB,QAAQ,iBAAiB;AAGtD,MAAM,uBAAuB;AAWtB,MAAM,yBAA0C;AAAA,EACrD,UAAU;AAAA,EACV,OAAO;AAAA,EACP,kBAAkB;AAAA,EAClB,mBAAmB,IAAI,MAAM,kBAAkB;AAAA,EAC/C,eAAe,MAAM;AAAA,EACrB,YAAY,MAAM;AACpB;AAmBO,MAAM,8BAA+B,aAAuD;AAAA,EACjG;AAAA,EACA;AAAA,EAEA,UAAU;AAAA,EACV,eAAe;AAAA,EACf,YAAY,IAAI,OAAO;AAAA,EAEvB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EAEpB,gBAAgB,IAAI,mBAA2B;AAAA,EAC/C,SAAmC,CAAC;AAAA,EACpC,iBAAiB,IAAI,mBAA2B;AAAA,EAChD,UAAqC,CAAC;AAAA,EAEtC,cAAc;AAAA,EACd;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,YAAY,MAAuB;AACjC,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,QAAQ;AAAA,EAC7B;AAAA,EAEA,UAAU,OAAmB;AAC3B,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,EAAE,gBAAgB,GAAG,MAAM,MAAM;AACnD,WAAK,QAAQ,KAAK,KAAK,UAAU;AACjC,WAAK,eAAe,IAAI,CAAC;AAAA,IAC3B;AACA,SAAK,WAAW,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACpE;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AAAA,QACf,gBAAgB,KAAK,MAAM,kBAAkB,OAAO;AAAA,QACpD,YAAY;AAAA,QACZ,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,oBAAoB;AAAA,MACtB;AACA,WAAK,OAAO,KAAK,KAAK,SAAS;AAC/B,WAAK,cAAc,IAAI,CAAC;AAAA,IAC1B;AAEA,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,eAAe,SAAS,IAAI;AAAA,EAC7C;AAAA,EAEA,sBAAsB;AACpB,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AAEpB,WAAK,UAAU,IAAI,WAAW,IAAI,WAAW,GAAG,MAAO,GAAG,CAAC,CAAC;AAAA,IAC9D;AAEA,SAAK,WAAY,OAAO;AACxB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,qBAAqB;AAtHvB;AAuHI,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,SAAS,EAAE;AAAA,IAClB;AAEA,SAAK,UAAW,OAAO;AACvB,eAAK,cAAL,mBAAgB,eAAe;AAC/B,eAAK,cAAL,mBAAgB,eAAe;AAC/B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,wBAAwB;AACtB,SAAK,gBAAgB;AACrB,SAAK;AAEL,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,KAAK,UAAU;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,yBAAyB;AACvB,SAAK,gBAAgB;AACrB,SAAK;AAAA,EACP;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,WAAoB;AAC9B,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AACA,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,SAAK,UAAU,QAAQ;AAEvB,eAAW,YAAY,KAAK,QAAQ;AAClC,2CAAU,eAAe;AAAA,IAC3B;AAEA,SAAK,OAAO,KAAK,MAAS;AAC1B,SAAK,QAAQ,KAAK,MAAS;AAC3B,SAAK,cAAc,IAAI,CAAC;AACxB,SAAK,eAAe,IAAI,CAAC;AAEzB,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,MAAM,YAAY;AAChB,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,CAAC,OAAO;AACb,YAAM,KAAK,cAAc,KAAK;AAC9B,YAAM,KAAK,eAAe,KAAK;AAE/B,aAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,QAAQ;AAChD,cAAM,WAAW,KAAK,OAAO,IAAI;AACjC,cAAM,YAAY,KAAK,QAAQ,IAAI;AAEnC,YAAI,EAAE,YAAY,YAAY;AAC5B,kBAAQ;AACR;AAAA,QACF;AAGA,eAAO,CAAC,KAAK,SAAS;AACpB,cAAI,KAAK,oBAAoB,SAAU;AACvC,gBAAM,KAAK,kBAAkB,GAAG;AAAA,QAClC;AAEA,cAAM,iBAAiB,SAAS;AAChC,cAAM,mBAAmB,KAAK,IAAI;AAElC,yBAAiB,MAAM,gBAAgB;AACrC,gBAAM,KAAK,cAAc,UAAU,kBAAkB,UAAU,WAAW,GAAG,KAAK;AAAA,QACpF;AAEA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cACJ,UACA,cACA,UACA,WACA,UACA;AACA,QAAI;AACJ,QAAI,UAAU,iBAAiB,KAAK,UAAU,MAAM;AAClD,kBAAY,KAAK,aAAa,SAAS,UAAU,EAAE,SAAS,UAAU;AAAA,IACxE;AAEA,UAAM,QAAQ,UAAU,KAAK;AAC7B,UAAM,QAAQ,KAAK,MAAM,WAAW,QAAQ;AAC5C,UAAM,iBAA2B,CAAC;AAElC,UAAM,SAAS,KAAK;AAEpB,eAAW,CAAC,MAAM,GAAG,GAAG,KAAK,OAAO;AAClC,UAAI,YAAY,KAAK,kBAAmB;AACxC,UAAI,KAAK,aAAc;AAEvB,YAAM,cAAc,KAAK,MAAM,cAAc,IAAI,EAAE;AACnD,qBAAe,KAAK,IAAI;AAExB,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,OAAO,SAAS,MAAM,GAAG,GAAG;AAElC,UAAI,QAAQ,KAAK;AACjB,UAAI;AACJ,UAAI,WAAW;AACb,gBAAQ;AACR,cAAM,oBAAoB,SAAS,qBAAqB,KAAK,MAAM;AACnE,cAAM,aAAa,oBAAoB;AACvC,cAAM,gBAAgB,KAAK,MAAM,QAAQ,OAAO;AAChD,cAAM,KAAK,gBAAgB,SAAS,mBAAmB;AACvD,cAAM,gBAAgB,KAAK,IAAI,GAAG,cAAc,EAAE;AAClD,gBAAQ,gBAAgB;AAAA,MAC1B,OAAO;AACL,gBAAQ,cAAc;AAAA,MACxB;AAEA,YAAM,aAAa,KAAK,IAAI,QAAQ,GAAG,IAAI,KAAK;AAChD,YAAM,KAAK,kBAAkB,aAAa,GAAO;AAEjD,WAAK;AAAA,QACH;AAAA,QACA,IAAI,qBAAqB;AAAA,UACvB,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,OAAO;AAAA,UACP,UAAU,KAAK,MAAM;AAAA,QACvB,CAAC;AAAA,MACH;AAEA,WAAK,cAAc,GAAG,MAAM,IAAI,IAAI;AACpC,YAAM,KAAK,mBAAmB,QAAQ,cAAc,GAAO;AAC3D,eAAS,oBAAoB;AAAA,IAC/B;AAEA,SAAK;AAAA,MACH;AAAA,MACA,IAAI,qBAAqB;AAAA,QACvB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,WAAW,OAAO,CAAC;AAAA,QACnB,SAAS,OAAO,CAAC;AAAA,QACjB,OAAO;AAAA,QACP,UAAU,KAAK,MAAM;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,SAAK,cAAc,GAAG,MAAM,IAAI,QAAQ;AAExC,UAAM,KAAK,kBAAkB,KAAK,MAAM,gBAAgB;AACxD,aAAS;AAAA,EACX;AAAA,EAEA,MAAM,kBAAkB,OAAe;AACrC,UAAM,QAAQ,KAAK;AAAA,MACjB,KAAK,UAAU;AAAA,MACf,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EAEA,aAAa,MAAwB;AACnC,UAAM,UAAoB,CAAC;AAC3B,UAAM,QAAQ,KAAK,MAAM,WAAW,IAAI;AACxC,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,MAAM,cAAc,KAAK,CAAC,CAAC;AAC1C,cAAQ,KAAK,GAAG,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB;AAChB,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":["../src/transcription.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TranscriptionSegment } from '@livekit/protocol';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { basic } from './tokenize/index.js';\nimport type { SentenceStream, SentenceTokenizer } from './tokenize/tokenizer.js';\nimport { AsyncIterableQueue, Future, shortuuid } from './utils.js';\n\n// standard speech rate in hyphens/ms\nconst STANDARD_SPEECH_RATE = 3830;\n\nexport interface TextSyncOptions {\n  /** Language code for transcription. */\n  language: string;\n  /** Speech speed multiplier. */\n  speed: number;\n  /** Delay between sentences in milliseconds. */\n  newSentenceDelay: number;\n  /** Tokenizer for splitting text into sentences. */\n  sentenceTokenizer: SentenceTokenizer;\n  /** Function to hyphenate words. */\n  hyphenateWord: (word: string) => string[];\n  /** Function to split text into words with positions. */\n  splitWords: (words: string) => [string, number, number][];\n}\n\nexport const defaultTextSyncOptions: TextSyncOptions = {\n  language: '',\n  speed: 1,\n  newSentenceDelay: 400,\n  sentenceTokenizer: new basic.SentenceTokenizer(),\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n};\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedSentences: number;\n}\n\ntype SyncCallbacks = {\n  textUpdated: (text: TranscriptionSegment) => void;\n};\n\nexport class TextAudioSynchronizer extends (EventEmitter as new () => TypedEmitter<SyncCallbacks>) {\n  #opts: TextSyncOptions;\n  #speed: number;\n\n  #closed = false;\n  #interrupted = false;\n  #closeFut = new Future();\n\n  #playingSegIndex = -1;\n  #finishedSegIndex = -1;\n\n  #textQChanged = new AsyncIterableQueue<number>();\n  #textQ: (TextData | undefined)[] = [];\n  #audioQChanged = new AsyncIterableQueue<number>();\n  #audioQ: (AudioData | undefined)[] = [];\n\n  #playedText = '';\n  #task?: Promise<void>;\n\n  #audioData?: AudioData;\n  #textData?: TextData;\n\n  constructor(opts: TextSyncOptions) {\n    super();\n\n    this.#opts = opts;\n    this.#speed = opts.speed * STANDARD_SPEECH_RATE;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    this.#checkNotClosed();\n    if (!this.#audioData) {\n      this.#audioData = { pushedDuration: 0, done: false };\n      this.#audioQ.push(this.#audioData);\n      this.#audioQChanged.put(1);\n    }\n    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  pushText(text: string) {\n    this.#checkNotClosed();\n    if (!this.#textData) {\n      this.#textData = {\n        sentenceStream: this.#opts.sentenceTokenizer.stream(),\n        pushedText: '',\n        done: false,\n        forwardedHyphens: 0,\n        forwardedSentences: 0,\n      };\n      this.#textQ.push(this.#textData);\n      this.#textQChanged.put(1);\n    }\n\n    this.#textData.pushedText += text;\n    this.#textData.sentenceStream.pushText(text);\n  }\n\n  markAudioSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#audioData) {\n      // create empty audio data if none exists\n      this.pushAudio(new AudioFrame(new Int16Array(), 24000, 1, 0));\n    }\n\n    this.#audioData!.done = true;\n    this.#audioData = undefined;\n  }\n\n  markTextSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#textData) {\n      this.pushText('');\n    }\n\n    this.#textData!.done = true;\n    this.#textData?.sentenceStream.flush();\n    this.#textData?.sentenceStream.close();\n    this.#textData = undefined;\n  }\n\n  segmentPlayoutStarted() {\n    this.#checkNotClosed();\n    this.#playingSegIndex++;\n\n    if (!this.#task) {\n      this.#task = this.#mainLoop();\n    }\n  }\n\n  segmentPlayoutFinished() {\n    this.#checkNotClosed();\n    this.#finishedSegIndex++;\n  }\n\n  get playedText(): string {\n    return this.#playedText;\n  }\n\n  async close(interrupt: boolean) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#interrupted = interrupt;\n    this.#closeFut.resolve();\n\n    for (const textData of this.#textQ) {\n      textData?.sentenceStream.close();\n    }\n\n    this.#textQ.push(undefined);\n    this.#audioQ.push(undefined);\n    this.#textQChanged.put(1);\n    this.#audioQChanged.put(1);\n\n    await this.#task;\n  }\n\n  async #mainLoop() {\n    let segIndex = 0;\n    let qDone = false;\n\n    while (!qDone) {\n      await this.#textQChanged.next();\n      await this.#audioQChanged.next();\n\n      while (this.#textQ.length && this.#audioQ.length) {\n        const textData = this.#textQ.pop();\n        const audioData = this.#audioQ.pop();\n\n        if (!(textData && audioData)) {\n          qDone = true;\n          break;\n        }\n\n        // wait for segment to start playing\n        while (!this.#closed) {\n          if (this.#playingSegIndex >= segIndex) break;\n          await this.#sleepIfNotClosed(125);\n        }\n\n        const sentenceStream = textData.sentenceStream;\n        const forwardStartTime = Date.now();\n\n        for await (const ev of sentenceStream) {\n          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);\n        }\n\n        segIndex++;\n      }\n    }\n  }\n\n  async #syncSentence(\n    segIndex: number,\n    segStartTime: number,\n    textData: TextData,\n    audioData: AudioData,\n    sentence: string,\n  ) {\n    let realSpeed: number | undefined;\n    if (audioData.pushedDuration > 0 && audioData.done) {\n      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;\n    }\n\n    const segId = shortuuid('SG_');\n    const words = this.#opts.splitWords(sentence);\n    const processedWords: string[] = [];\n\n    const ogText = this.#playedText;\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    for (const [word, _, end] of words) {\n      if (segIndex <= this.#finishedSegIndex) break;\n      if (this.#interrupted) return;\n\n      const wordHyphens = this.#opts.hyphenateWord(word).length;\n      processedWords.push(word);\n\n      const elapsed = Date.now() - segStartTime;\n      const text = sentence.slice(0, end); // TODO: rstrip punctuations\n\n      let speed = this.#speed;\n      let delay: number;\n      if (realSpeed) {\n        speed = realSpeed;\n        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;\n        const hyphPauses = estimatedPausesMs * speed;\n        const targetHyphens = Math.round(speed * elapsed);\n        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;\n        const toWaitHyphens = Math.max(0, wordHyphens - dt);\n        delay = toWaitHyphens / speed;\n      } else {\n        delay = wordHyphens / speed;\n      }\n\n      const firstDelay = Math.min(delay / 2, 2 / speed);\n      await this.#sleepIfNotClosed(firstDelay * 1000000);\n\n      this.emit(\n        'textUpdated',\n        new TranscriptionSegment({\n          id: segId,\n          text: text,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          final: false,\n          language: this.#opts.language,\n        }),\n      );\n\n      this.#playedText = `${ogText} ${text}`;\n      await this.#sleepIfNotClosed((delay - firstDelay) * 1000000);\n      textData.forwardedHyphens += wordHyphens;\n    }\n\n    this.emit(\n      'textUpdated',\n      new TranscriptionSegment({\n        id: segId,\n        text: sentence,\n        startTime: BigInt(0),\n        endTime: BigInt(0),\n        final: true,\n        language: this.#opts.language,\n      }),\n    );\n\n    this.#playedText = `${ogText} ${sentence}`;\n\n    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);\n    textData.forwardedSentences++;\n  }\n\n  async #sleepIfNotClosed(delay: number) {\n    await Promise.race([\n      this.#closeFut.await,\n      new Promise((resolve) => setTimeout(resolve, delay)),\n    ]);\n  }\n\n  #calcHyphens(text: string): string[] {\n    const hyphens: string[] = [];\n    const words = this.#opts.splitWords(text);\n    for (const word of words) {\n      const n = this.#opts.hyphenateWord(word[0]);\n      hyphens.push(...n);\n    }\n    return hyphens;\n  }\n\n  #checkNotClosed() {\n    if (this.#closed) {\n      throw new Error('TextAudioSynchronizer is closed');\n    }\n  }\n}\n"],"mappings":"AAGA,SAAS,4BAA4B;AACrC,SAAS,kBAAkB;AAE3B,SAAS,oBAAoB;AAC7B,SAAS,aAAa;AAEtB,SAAS,oBAAoB,QAAQ,iBAAiB;AAGtD,MAAM,uBAAuB;AAiBtB,MAAM,yBAA0C;AAAA,EACrD,UAAU;AAAA,EACV,OAAO;AAAA,EACP,kBAAkB;AAAA,EAClB,mBAAmB,IAAI,MAAM,kBAAkB;AAAA,EAC/C,eAAe,MAAM;AAAA,EACrB,YAAY,MAAM;AACpB;AAmBO,MAAM,8BAA+B,aAAuD;AAAA,EACjG;AAAA,EACA;AAAA,EAEA,UAAU;AAAA,EACV,eAAe;AAAA,EACf,YAAY,IAAI,OAAO;AAAA,EAEvB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EAEpB,gBAAgB,IAAI,mBAA2B;AAAA,EAC/C,SAAmC,CAAC;AAAA,EACpC,iBAAiB,IAAI,mBAA2B;AAAA,EAChD,UAAqC,CAAC;AAAA,EAEtC,cAAc;AAAA,EACd;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,YAAY,MAAuB;AACjC,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,QAAQ;AAAA,EAC7B;AAAA,EAEA,UAAU,OAAmB;AAC3B,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,EAAE,gBAAgB,GAAG,MAAM,MAAM;AACnD,WAAK,QAAQ,KAAK,KAAK,UAAU;AACjC,WAAK,eAAe,IAAI,CAAC;AAAA,IAC3B;AACA,SAAK,WAAW,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACpE;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AAAA,QACf,gBAAgB,KAAK,MAAM,kBAAkB,OAAO;AAAA,QACpD,YAAY;AAAA,QACZ,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,oBAAoB;AAAA,MACtB;AACA,WAAK,OAAO,KAAK,KAAK,SAAS;AAC/B,WAAK,cAAc,IAAI,CAAC;AAAA,IAC1B;AAEA,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,eAAe,SAAS,IAAI;AAAA,EAC7C;AAAA,EAEA,sBAAsB;AACpB,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AAEpB,WAAK,UAAU,IAAI,WAAW,IAAI,WAAW,GAAG,MAAO,GAAG,CAAC,CAAC;AAAA,IAC9D;AAEA,SAAK,WAAY,OAAO;AACxB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,qBAAqB;AA5HvB;AA6HI,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,SAAS,EAAE;AAAA,IAClB;AAEA,SAAK,UAAW,OAAO;AACvB,eAAK,cAAL,mBAAgB,eAAe;AAC/B,eAAK,cAAL,mBAAgB,eAAe;AAC/B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,wBAAwB;AACtB,SAAK,gBAAgB;AACrB,SAAK;AAEL,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,KAAK,UAAU;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,yBAAyB;AACvB,SAAK,gBAAgB;AACrB,SAAK;AAAA,EACP;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,WAAoB;AAC9B,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AACA,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,SAAK,UAAU,QAAQ;AAEvB,eAAW,YAAY,KAAK,QAAQ;AAClC,2CAAU,eAAe;AAAA,IAC3B;AAEA,SAAK,OAAO,KAAK,MAAS;AAC1B,SAAK,QAAQ,KAAK,MAAS;AAC3B,SAAK,cAAc,IAAI,CAAC;AACxB,SAAK,eAAe,IAAI,CAAC;AAEzB,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,MAAM,YAAY;AAChB,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,CAAC,OAAO;AACb,YAAM,KAAK,cAAc,KAAK;AAC9B,YAAM,KAAK,eAAe,KAAK;AAE/B,aAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,QAAQ;AAChD,cAAM,WAAW,KAAK,OAAO,IAAI;AACjC,cAAM,YAAY,KAAK,QAAQ,IAAI;AAEnC,YAAI,EAAE,YAAY,YAAY;AAC5B,kBAAQ;AACR;AAAA,QACF;AAGA,eAAO,CAAC,KAAK,SAAS;AACpB,cAAI,KAAK,oBAAoB,SAAU;AACvC,gBAAM,KAAK,kBAAkB,GAAG;AAAA,QAClC;AAEA,cAAM,iBAAiB,SAAS;AAChC,cAAM,mBAAmB,KAAK,IAAI;AAElC,yBAAiB,MAAM,gBAAgB;AACrC,gBAAM,KAAK,cAAc,UAAU,kBAAkB,UAAU,WAAW,GAAG,KAAK;AAAA,QACpF;AAEA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cACJ,UACA,cACA,UACA,WACA,UACA;AACA,QAAI;AACJ,QAAI,UAAU,iBAAiB,KAAK,UAAU,MAAM;AAClD,kBAAY,KAAK,aAAa,SAAS,UAAU,EAAE,SAAS,UAAU;AAAA,IACxE;AAEA,UAAM,QAAQ,UAAU,KAAK;AAC7B,UAAM,QAAQ,KAAK,MAAM,WAAW,QAAQ;AAC5C,UAAM,iBAA2B,CAAC;AAElC,UAAM,SAAS,KAAK;AAEpB,eAAW,CAAC,MAAM,GAAG,GAAG,KAAK,OAAO;AAClC,UAAI,YAAY,KAAK,kBAAmB;AACxC,UAAI,KAAK,aAAc;AAEvB,YAAM,cAAc,KAAK,MAAM,cAAc,IAAI,EAAE;AACnD,qBAAe,KAAK,IAAI;AAExB,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,OAAO,SAAS,MAAM,GAAG,GAAG;AAElC,UAAI,QAAQ,KAAK;AACjB,UAAI;AACJ,UAAI,WAAW;AACb,gBAAQ;AACR,cAAM,oBAAoB,SAAS,qBAAqB,KAAK,MAAM;AACnE,cAAM,aAAa,oBAAoB;AACvC,cAAM,gBAAgB,KAAK,MAAM,QAAQ,OAAO;AAChD,cAAM,KAAK,gBAAgB,SAAS,mBAAmB;AACvD,cAAM,gBAAgB,KAAK,IAAI,GAAG,cAAc,EAAE;AAClD,gBAAQ,gBAAgB;AAAA,MAC1B,OAAO;AACL,gBAAQ,cAAc;AAAA,MACxB;AAEA,YAAM,aAAa,KAAK,IAAI,QAAQ,GAAG,IAAI,KAAK;AAChD,YAAM,KAAK,kBAAkB,aAAa,GAAO;AAEjD,WAAK;AAAA,QACH;AAAA,QACA,IAAI,qBAAqB;AAAA,UACvB,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,OAAO;AAAA,UACP,UAAU,KAAK,MAAM;AAAA,QACvB,CAAC;AAAA,MACH;AAEA,WAAK,cAAc,GAAG,MAAM,IAAI,IAAI;AACpC,YAAM,KAAK,mBAAmB,QAAQ,cAAc,GAAO;AAC3D,eAAS,oBAAoB;AAAA,IAC/B;AAEA,SAAK;AAAA,MACH;AAAA,MACA,IAAI,qBAAqB;AAAA,QACvB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,WAAW,OAAO,CAAC;AAAA,QACnB,SAAS,OAAO,CAAC;AAAA,QACjB,OAAO;AAAA,QACP,UAAU,KAAK,MAAM;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,SAAK,cAAc,GAAG,MAAM,IAAI,QAAQ;AAExC,UAAM,KAAK,kBAAkB,KAAK,MAAM,gBAAgB;AACxD,aAAS;AAAA,EACX;AAAA,EAEA,MAAM,kBAAkB,OAAe;AACrC,UAAM,QAAQ,KAAK;AAAA,MACjB,KAAK,UAAU;AAAA,MACf,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EAEA,aAAa,MAAwB;AACnC,UAAM,UAAoB,CAAC;AAC3B,UAAM,QAAQ,KAAK,MAAM,WAAW,IAAI;AACxC,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,MAAM,cAAc,KAAK,CAAC,CAAC;AAC1C,cAAQ,KAAK,GAAG,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB;AAChB,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}

package/dist/tts/fallback_adapter.cjs

@@ -0,0 +1,466 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var fallback_adapter_exports = {};
+__export(fallback_adapter_exports, {
+  FallbackAdapter: () => FallbackAdapter
+});
+module.exports = __toCommonJS(fallback_adapter_exports);
+var import_rtc_node = require("@livekit/rtc-node");
+var import_exceptions = require("../_exceptions.cjs");
+var import_log = require("../log.cjs");
+var import_tokenize = require("../tokenize/index.cjs");
+var import_types = require("../types.cjs");
+var import_utils = require("../utils.cjs");
+var import_stream_adapter = require("./stream_adapter.cjs");
+var import_tts = require("./tts.cjs");
+const DEFAULT_FALLBACK_API_CONNECT_OPTIONS = {
+  maxRetry: 0,
+  timeoutMs: import_types.DEFAULT_API_CONNECT_OPTIONS.timeoutMs,
+  retryIntervalMs: import_types.DEFAULT_API_CONNECT_OPTIONS.retryIntervalMs
+};
+const FORWARD_POLL_MS = 10;
+class FallbackAdapter extends import_tts.TTS {
+  /** The list of TTS instances used for fallback (in priority order). */
+  ttsInstances;
+  /** Number of retries per TTS instance before falling back to the next one. */
+  maxRetryPerTTS;
+  /** Delay in milliseconds before attempting to recover a failed TTS instance. */
+  recoveryDelayMs;
+  _status = [];
+  _logger = (0, import_log.log)();
+  _recoveryTimeouts = /* @__PURE__ */ new Map();
+  label = `tts.FallbackAdapter`;
+  constructor(opts) {
+    if (!opts.ttsInstances || opts.ttsInstances.length < 1) {
+      throw new Error("at least one TTS instance must be provided.");
+    }
+    const numChannels = opts.ttsInstances[0].numChannels;
+    const allNumChannelsMatch = opts.ttsInstances.every((tts) => tts.numChannels === numChannels);
+    if (!allNumChannelsMatch) {
+      throw new Error("All TTS instances should have the same number of channels");
+    }
+    const sampleRate = Math.max(...opts.ttsInstances.map((t) => t.sampleRate));
+    const capabilities = FallbackAdapter.aggregateCapabilities(opts.ttsInstances);
+    super(sampleRate, numChannels, capabilities);
+    this.ttsInstances = opts.ttsInstances;
+    this.maxRetryPerTTS = opts.maxRetryPerTTS ?? 2;
+    this.recoveryDelayMs = opts.recoveryDelayMs ?? 1e3;
+    this._status = opts.ttsInstances.map(() => ({
+      available: true,
+      recoveringTask: null
+    }));
+    this.setupEventForwarding();
+  }
+  static aggregateCapabilities(instances) {
+    const streaming = instances.some((tts) => tts.capabilities.streaming);
+    const alignedTranscript = instances.every((tts) => tts.capabilities.alignedTranscript === true);
+    return { streaming, alignedTranscript };
+  }
+  setupEventForwarding() {
+    this.ttsInstances.forEach((tts) => {
+      tts.on("metrics_collected", (metrics) => {
+        this.emit("metrics_collected", metrics);
+      });
+      tts.on("error", (error) => {
+        this.emit("error", error);
+      });
+    });
+  }
+  /**
+   * Returns the current status of all TTS instances, including availability and recovery state.
+   */
+  get status() {
+    return this._status;
+  }
+  getStreamingInstance(index) {
+    const tts = this.ttsInstances[index];
+    if (tts.capabilities.streaming) {
+      return tts;
+    }
+    return new import_stream_adapter.StreamAdapter(tts, new import_tokenize.basic.SentenceTokenizer());
+  }
+  /**
+   * Creates a new AudioResampler for the given TTS index if needed.
+   * Returns null if the TTS sample rate matches the adapter's output rate.
+   * Each stream should create its own resampler to avoid concurrency issues.
+   * @internal
+   */
+  createResamplerForTTS(index) {
+    const tts = this.ttsInstances[index];
+    if (this.sampleRate !== tts.sampleRate) {
+      this._logger.debug(
+        `resampling ${tts.label} from ${tts.sampleRate}Hz to ${this.sampleRate}Hz`
+      );
+      return new import_rtc_node.AudioResampler(tts.sampleRate, this.sampleRate, tts.numChannels);
+    }
+    return null;
+  }
+  emitAvailabilityChanged(tts, available) {
+    const event = { tts, available };
+    this.emit(
+      "tts_availability_changed",
+      event
+    );
+  }
+  tryRecovery(index) {
+    const status = this._status[index];
+    const tts = this.ttsInstances[index];
+    if (status.recoveringTask && !status.recoveringTask.done) {
+      return;
+    }
+    status.recoveringTask = import_utils.Task.from(async (controller) => {
+      try {
+        const testStream = tts.synthesize(
+          "Hello world, this is a recovery test.",
+          {
+            maxRetry: 0,
+            timeoutMs: 1e4,
+            retryIntervalMs: 1e3
+          },
+          controller.signal
+        );
+        let audioReceived = false;
+        for await (const _ of testStream) {
+          audioReceived = true;
+        }
+        if (!audioReceived) {
+          throw new Error("Recovery test completed but no audio was received");
+        }
+        status.available = true;
+        status.recoveringTask = null;
+        this._logger.info({ tts: tts.label }, "TTS recovered");
+        this.emitAvailabilityChanged(tts, true);
+      } catch (error) {
+        status.recoveringTask = null;
+        if (controller.signal.aborted) {
+          return;
+        }
+        this._logger.debug({ tts: tts.label, error }, "TTS recovery failed, will retry");
+        const timeoutId = setTimeout(() => {
+          this._recoveryTimeouts.delete(index);
+          this.tryRecovery(index);
+        }, this.recoveryDelayMs);
+        this._recoveryTimeouts.set(index, timeoutId);
+      }
+    });
+  }
+  markUnAvailable(index) {
+    const status = this._status[index];
+    if (status.recoveringTask && !status.recoveringTask.done) {
+      return;
+    }
+    if (status.available) {
+      status.available = false;
+      this.emitAvailabilityChanged(this.ttsInstances[index], false);
+    }
+    this.tryRecovery(index);
+  }
+  /**
+   * Receives text and returns synthesis in the form of a {@link ChunkedStream}
+   */
+  synthesize(text, connOptions, abortSignal) {
+    return new FallbackChunkedStream(
+      this,
+      text,
+      connOptions ?? DEFAULT_FALLBACK_API_CONNECT_OPTIONS,
+      abortSignal
+    );
+  }
+  /**
+   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
+   *
+   * @param options - Optional configuration including connection options
+   */
+  stream(options) {
+    return new FallbackSynthesizeStream(
+      this,
+      (options == null ? void 0 : options.connOptions) ?? DEFAULT_FALLBACK_API_CONNECT_OPTIONS
+    );
+  }
+  /**
+   * Close the FallbackAdapter and all underlying TTS instances.
+   * This cancels any ongoing recovery tasks and cleans up resources.
+   */
+  async close() {
+    this._recoveryTimeouts.forEach((timeoutId) => {
+      clearTimeout(timeoutId);
+    });
+    this._recoveryTimeouts.clear();
+    const recoveryTasks = this._status.map((s) => s.recoveringTask).filter((t) => t !== null);
+    if (recoveryTasks.length > 0) {
+      await (0, import_utils.cancelAndWait)(recoveryTasks, 1e3);
+    }
+    for (const tts of this.ttsInstances) {
+      tts.removeAllListeners("metrics_collected");
+      tts.removeAllListeners("error");
+    }
+    await Promise.all(this.ttsInstances.map((tts) => tts.close()));
+  }
+}
+class FallbackChunkedStream extends import_tts.ChunkedStream {
+  adapter;
+  connOptions;
+  _logger = (0, import_log.log)();
+  label = "tts.FallbackChunkedStream";
+  constructor(adapter, text, connOptions, abortSignal) {
+    super(text, adapter, connOptions, abortSignal);
+    this.adapter = adapter;
+    this.connOptions = connOptions;
+  }
+  async run() {
+    const allTTSFailed = this.adapter.status.every((s) => !s.available);
+    let lastRequestId = "";
+    let lastSegmentId = "";
+    if (allTTSFailed) {
+      this._logger.warn("All fallback TTS instances failed, retrying from first...");
+    }
+    for (let i = 0; i < this.adapter.ttsInstances.length; i++) {
+      const tts = this.adapter.ttsInstances[i];
+      const status = this.adapter.status[i];
+      if (!status.available && !allTTSFailed) {
+        this.adapter.markUnAvailable(i);
+        continue;
+      }
+      try {
+        this._logger.debug({ tts: tts.label }, "attempting TTS synthesis");
+        const connOptions = {
+          ...this.connOptions,
+          maxRetry: this.adapter.maxRetryPerTTS
+        };
+        const stream = tts.synthesize(this.inputText, connOptions, this.abortSignal);
+        let audioReceived = false;
+        const resampler = this.adapter.createResamplerForTTS(i);
+        for await (const audio of stream) {
+          if (this.abortController.signal.aborted) {
+            stream.close();
+            return;
+          }
+          if (resampler) {
+            for (const frame of resampler.push(audio.frame)) {
+              this.queue.put({
+                ...audio,
+                frame
+              });
+              audioReceived = true;
+            }
+          } else {
+            this.queue.put(audio);
+            audioReceived = true;
+          }
+          lastRequestId = audio.requestId;
+          lastSegmentId = audio.segmentId;
+        }
+        if (resampler) {
+          for (const frame of resampler.flush()) {
+            this.queue.put({
+              requestId: lastRequestId || "",
+              segmentId: lastSegmentId || "",
+              frame,
+              final: true
+            });
+            audioReceived = true;
+          }
+        }
+        if (!audioReceived) {
+          throw new import_exceptions.APIConnectionError({
+            message: "TTS synthesis completed but no audio was received"
+          });
+        }
+        this._logger.debug({ tts: tts.label }, "TTS synthesis succeeded");
+        return;
+      } catch (error) {
+        if (error instanceof import_exceptions.APIError || error instanceof import_exceptions.APIConnectionError) {
+          this._logger.warn({ tts: tts.label, error }, "TTS failed, switching to next instance");
+          this.adapter.markUnAvailable(i);
+        } else {
+          throw error;
+        }
+      }
+    }
+    const labels = this.adapter.ttsInstances.map((t) => t.label).join(", ");
+    throw new import_exceptions.APIConnectionError({
+      message: `all TTS instances failed (${labels})`
+    });
+  }
+}
+class FallbackSynthesizeStream extends import_tts.SynthesizeStream {
+  adapter;
+  tokenBuffer = [];
+  audioPushed = false;
+  _logger = (0, import_log.log)();
+  label = "tts.FallbackSynthesizeStream";
+  constructor(adapter, connOptions) {
+    super(adapter, connOptions);
+    this.adapter = adapter;
+  }
+  async run() {
+    const allTTSFailed = this.adapter.status.every((s) => !s.available);
+    if (allTTSFailed) {
+      this._logger.warn("All fallback TTS instances failed, retrying from first...");
+    }
+    const readInputLLMStream = (async () => {
+      try {
+        for await (const input of this.input) {
+          if (this.abortController.signal.aborted) break;
+          this.tokenBuffer.push(input);
+        }
+      } catch (error) {
+        this._logger.debug({ error }, "Error reading input LLM stream");
+        throw error;
+      } finally {
+        this.tokenBuffer.push(import_tts.SynthesizeStream.END_OF_STREAM);
+      }
+    })();
+    for (let i = 0; i < this.adapter.ttsInstances.length; i++) {
+      const tts = this.adapter.getStreamingInstance(i);
+      const originalTts = this.adapter.ttsInstances[i];
+      const status = this.adapter.status[i];
+      let lastRequestId = "";
+      let lastSegmentId = "";
+      if (!status.available && !allTTSFailed) {
+        this.adapter.markUnAvailable(i);
+        continue;
+      }
+      try {
+        this._logger.debug({ tts: originalTts.label }, "attempting TTS stream");
+        const connOptions = {
+          ...this.connOptions,
+          maxRetry: this.adapter.maxRetryPerTTS
+        };
+        const stream = tts.stream({ connOptions });
+        const resampler = this.adapter.createResamplerForTTS(i);
+        let bufferIndex = 0;
+        let streamOutputCompleted = false;
+        const forwardBufferToTTS = async () => {
+          while (true) {
+            while (bufferIndex < this.tokenBuffer.length) {
+              const token = this.tokenBuffer[bufferIndex++];
+              if (token === import_tts.SynthesizeStream.FLUSH_SENTINEL) {
+                stream.flush();
+              } else if (token === import_tts.SynthesizeStream.END_OF_STREAM) {
+                stream.endInput();
+                return;
+              } else {
+                stream.pushText(token);
+              }
+            }
+            await new Promise((resolve) => setTimeout(resolve, FORWARD_POLL_MS));
+            if (this.abortController.signal.aborted || streamOutputCompleted) {
+              stream.endInput();
+              return;
+            }
+          }
+        };
+        const processOutput = async () => {
+          try {
+            for await (const audio of stream) {
+              if (this.abortController.signal.aborted) {
+                stream.close();
+                return;
+              }
+              if (audio === import_tts.SynthesizeStream.END_OF_STREAM) {
+                continue;
+              }
+              if (resampler) {
+                for (const frame of resampler.push(audio.frame)) {
+                  this.queue.put({
+                    ...audio,
+                    frame
+                  });
+                  this.audioPushed = true;
+                }
+              } else {
+                this.queue.put(audio);
+                this.audioPushed = true;
+              }
+              lastRequestId = audio.requestId;
+              lastSegmentId = audio.segmentId;
+            }
+            if (resampler) {
+              for (const frame of resampler.flush()) {
+                this.queue.put({
+                  requestId: lastRequestId || "",
+                  segmentId: lastSegmentId || "",
+                  frame,
+                  final: true
+                });
+                this.audioPushed = true;
+              }
+            }
+          } finally {
+            streamOutputCompleted = true;
+          }
+        };
+        const [outputResult, forwardBufferResult] = await Promise.allSettled([
+          processOutput(),
+          forwardBufferToTTS().catch((err) => {
+            stream.close();
+            throw err;
+          })
+        ]);
+        if (outputResult.status === "rejected") {
+          stream.close();
+          throw outputResult.reason;
+        }
+        if (forwardBufferResult.status === "rejected") {
+          stream.close();
+          throw forwardBufferResult.reason;
+        }
+        if (!this.audioPushed) {
+          throw new import_exceptions.APIConnectionError({
+            message: "TTS stream completed but no audio was received"
+          });
+        }
+        this.queue.put(import_tts.SynthesizeStream.END_OF_STREAM);
+        this._logger.debug({ tts: originalTts.label }, "TTS stream succeeded");
+        await readInputLLMStream.catch(() => {
+        });
+        return;
+      } catch (error) {
+        if (this.audioPushed) {
+          this._logger.error(
+            { tts: originalTts.label },
+            "TTS failed after audio pushed, cannot fallback mid-utterance"
+          );
+          throw error;
+        }
+        if (error instanceof import_exceptions.APIError || error instanceof import_exceptions.APIConnectionError) {
+          this._logger.warn(
+            { tts: originalTts.label, error },
+            "TTS failed, switching to next instance"
+          );
+          this.adapter.markUnAvailable(i);
+        } else {
+          throw error;
+        }
+      }
+    }
+    await readInputLLMStream.catch(() => {
+    });
+    const labels = this.adapter.ttsInstances.map((t) => t.label).join(", ");
+    throw new import_exceptions.APIConnectionError({
+      message: `all TTS instances failed (${labels})`
+    });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  FallbackAdapter
+});
+//# sourceMappingURL=fallback_adapter.cjs.map

package/dist/tts/fallback_adapter.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tts/fallback_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioResampler } from '@livekit/rtc-node';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport { basic } from '../tokenize/index.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { Task, cancelAndWait } from '../utils.js';\nimport { StreamAdapter } from './stream_adapter.js';\nimport { ChunkedStream, SynthesizeStream, TTS, type TTSCapabilities } from './tts.js';\n\n/**\n * Internal status tracking for each TTS instance.\n * @internal\n */\ninterface TTSStatus {\n  available: boolean;\n  recoveringTask: Task<void> | null;\n}\n\n/**\n * Options for creating a FallbackAdapter.\n */\nexport interface FallbackAdapterOptions {\n  /** List of TTS instances to use for fallback (in priority order). At least one is required. */\n  ttsInstances: TTS[];\n  /** Number of internal retries per TTS instance before moving to the next one. Defaults to 2. */\n  maxRetryPerTTS?: number;\n  /** Delay in milliseconds before attempting to recover a failed TTS instance. Defaults to 1000. */\n  recoveryDelayMs?: number;\n}\n\n/**\n * Event emitted when a TTS instance's availability changes.\n */\nexport interface AvailabilityChangedEvent {\n  /** The TTS instance whose availability changed. */\n  tts: TTS;\n  /** Whether the TTS instance is now available. */\n  available: boolean;\n}\n\nconst DEFAULT_FALLBACK_API_CONNECT_OPTIONS: APIConnectOptions = {\n  maxRetry: 0,\n  timeoutMs: DEFAULT_API_CONNECT_OPTIONS.timeoutMs,\n  retryIntervalMs: DEFAULT_API_CONNECT_OPTIONS.retryIntervalMs,\n};\n\nconst FORWARD_POLL_MS = 10;\n\n/**\n * FallbackAdapter is a TTS wrapper that provides automatic failover between multiple TTS providers.\n *\n * When the primary TTS fails, it automatically switches to the next available provider in the list.\n * Failed providers are monitored in the background and restored when they recover.\n *\n * Features:\n * - Automatic failover to backup TTS providers on failure\n * - Background health checks to restore recovered providers\n * - Automatic audio resampling when TTS providers have different sample rates\n * - Support for both streaming and non-streaming TTS providers\n *\n * @example\n * ```typescript\n * import { FallbackAdapter } from '@livekit/agents';\n * import { TTS as OpenAITTS } from '@livekit/agents-plugin-openai';\n * import { TTS as ElevenLabsTTS } from '@livekit/agents-plugin-elevenlabs';\n *\n * const fallbackTTS = new FallbackAdapter({\n *   ttsInstances: [\n *     new OpenAITTS(),      // Primary\n *     new ElevenLabsTTS(),  // Fallback\n *   ],\n *   maxRetryPerTTS: 2,      // Retry each TTS twice before moving to next\n *   recoveryDelayMs: 1000,  // Check recovery every 1 second\n * });\n *\n * ```\n */\nexport class FallbackAdapter extends TTS {\n  /** The list of TTS instances used for fallback (in priority order). */\n  readonly ttsInstances: TTS[];\n  /** Number of retries per TTS instance before falling back to the next one. */\n  readonly maxRetryPerTTS: number;\n  /** Delay in milliseconds before attempting to recover a failed TTS instance. */\n  readonly recoveryDelayMs: number;\n\n  private _status: TTSStatus[] = [];\n  private _logger = log();\n  private _recoveryTimeouts: Map<number, NodeJS.Timeout> = new Map();\n\n  label: string = `tts.FallbackAdapter`;\n\n  constructor(opts: FallbackAdapterOptions) {\n    if (!opts.ttsInstances || opts.ttsInstances.length < 1) {\n      throw new Error('at least one TTS instance must be provided.');\n    }\n    const numChannels = opts.ttsInstances[0]!.numChannels;\n    const allNumChannelsMatch = opts.ttsInstances.every((tts) => tts.numChannels === numChannels);\n    if (!allNumChannelsMatch) {\n      throw new Error('All TTS instances should have the same number of channels');\n    }\n    const sampleRate = Math.max(...opts.ttsInstances.map((t) => t.sampleRate));\n    const capabilities = FallbackAdapter.aggregateCapabilities(opts.ttsInstances);\n    super(sampleRate, numChannels, capabilities);\n    this.ttsInstances = opts.ttsInstances;\n    this.maxRetryPerTTS = opts.maxRetryPerTTS ?? 2;\n    this.recoveryDelayMs = opts.recoveryDelayMs ?? 1000;\n    this._status = opts.ttsInstances.map(() => ({\n      available: true,\n      recoveringTask: null,\n    }));\n    this.setupEventForwarding();\n  }\n  private static aggregateCapabilities(instances: TTS[]): TTSCapabilities {\n    const streaming = instances.some((tts) => tts.capabilities.streaming);\n    const alignedTranscript = instances.every((tts) => tts.capabilities.alignedTranscript === true);\n    return { streaming, alignedTranscript };\n  }\n\n  private setupEventForwarding(): void {\n    this.ttsInstances.forEach((tts) => {\n      tts.on('metrics_collected', (metrics) => {\n        this.emit('metrics_collected', metrics);\n      });\n      tts.on('error', (error) => {\n        this.emit('error', error);\n      });\n    });\n  }\n\n  /**\n   * Returns the current status of all TTS instances, including availability and recovery state.\n   */\n  get status(): TTSStatus[] {\n    return this._status;\n  }\n\n  getStreamingInstance(index: number): TTS {\n    const tts = this.ttsInstances[index]!;\n    if (tts.capabilities.streaming) {\n      return tts;\n    }\n    // Wrap non-streaming TTS with StreamAdapter\n    return new StreamAdapter(tts, new basic.SentenceTokenizer());\n  }\n\n  /**\n   * Creates a new AudioResampler for the given TTS index if needed.\n   * Returns null if the TTS sample rate matches the adapter's output rate.\n   * Each stream should create its own resampler to avoid concurrency issues.\n   * @internal\n   */\n  createResamplerForTTS(index: number): AudioResampler | null {\n    const tts = this.ttsInstances[index]!;\n    if (this.sampleRate !== tts.sampleRate) {\n      this._logger.debug(\n        `resampling ${tts.label} from ${tts.sampleRate}Hz to ${this.sampleRate}Hz`,\n      );\n      return new AudioResampler(tts.sampleRate, this.sampleRate, tts.numChannels);\n    }\n    return null;\n  }\n\n  private emitAvailabilityChanged(tts: TTS, available: boolean): void {\n    const event: AvailabilityChangedEvent = { tts, available };\n    (this as unknown as { emit: (event: string, data: AvailabilityChangedEvent) => void }).emit(\n      'tts_availability_changed',\n      event,\n    );\n  }\n\n  private tryRecovery(index: number): void {\n    const status = this._status[index]!;\n    const tts = this.ttsInstances[index]!;\n    if (status.recoveringTask && !status.recoveringTask.done) {\n      return;\n    }\n    status.recoveringTask = Task.from(async (controller) => {\n      try {\n        const testStream = tts.synthesize(\n          'Hello world, this is a recovery test.',\n          {\n            maxRetry: 0,\n            timeoutMs: 10000,\n            retryIntervalMs: 1000,\n          },\n          controller.signal,\n        );\n        let audioReceived = false;\n        for await (const _ of testStream) {\n          audioReceived = true;\n        }\n        if (!audioReceived) {\n          throw new Error('Recovery test completed but no audio was received');\n        }\n\n        status.available = true;\n        status.recoveringTask = null;\n        this._logger.info({ tts: tts.label }, 'TTS recovered');\n        this.emitAvailabilityChanged(tts, true);\n      } catch (error) {\n        status.recoveringTask = null;\n        // Don't schedule retry if we're shutting down\n        if (controller.signal.aborted) {\n          return;\n        }\n        this._logger.debug({ tts: tts.label, error }, 'TTS recovery failed, will retry');\n        // Retry recovery after delay (matches Python's retry behavior)\n        const timeoutId = setTimeout(() => {\n          this._recoveryTimeouts.delete(index);\n          this.tryRecovery(index);\n        }, this.recoveryDelayMs);\n        this._recoveryTimeouts.set(index, timeoutId);\n      }\n    });\n  }\n\n  markUnAvailable(index: number): void {\n    const status = this._status[index]!;\n    if (status.recoveringTask && !status.recoveringTask.done) {\n      return;\n    }\n    if (status.available) {\n      status.available = false;\n      this.emitAvailabilityChanged(this.ttsInstances[index]!, false);\n    }\n    this.tryRecovery(index);\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  synthesize(\n    text: string,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ): ChunkedStream {\n    return new FallbackChunkedStream(\n      this,\n      text,\n      connOptions ?? DEFAULT_FALLBACK_API_CONNECT_OPTIONS,\n      abortSignal,\n    );\n  }\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   *\n   * @param options - Optional configuration including connection options\n   */\n  stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream {\n    return new FallbackSynthesizeStream(\n      this,\n      options?.connOptions ?? DEFAULT_FALLBACK_API_CONNECT_OPTIONS,\n    );\n  }\n\n  /**\n   * Close the FallbackAdapter and all underlying TTS instances.\n   * This cancels any ongoing recovery tasks and cleans up resources.\n   */\n  async close(): Promise<void> {\n    // clear all recovery timeouts so that it does not cause issue\n    this._recoveryTimeouts.forEach((timeoutId) => {\n      clearTimeout(timeoutId);\n    });\n    this._recoveryTimeouts.clear();\n\n    // Cancel all recovery tasks\n    const recoveryTasks = this._status\n      .map((s) => s.recoveringTask)\n      .filter((t): t is Task<void> => t !== null);\n\n    if (recoveryTasks.length > 0) {\n      await cancelAndWait(recoveryTasks, 1000);\n    }\n\n    // Remove event listeners\n    for (const tts of this.ttsInstances) {\n      tts.removeAllListeners('metrics_collected');\n      tts.removeAllListeners('error');\n    }\n\n    // Close all TTS instances\n    await Promise.all(this.ttsInstances.map((tts) => tts.close()));\n  }\n}\n\nclass FallbackChunkedStream extends ChunkedStream {\n  private adapter: FallbackAdapter;\n  private connOptions: APIConnectOptions;\n  private _logger = log();\n\n  label: string = 'tts.FallbackChunkedStream';\n\n  constructor(\n    adapter: FallbackAdapter,\n    text: string,\n    connOptions: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ) {\n    super(text, adapter, connOptions, abortSignal);\n    this.adapter = adapter;\n    this.connOptions = connOptions;\n  }\n\n  protected async run(): Promise<void> {\n    const allTTSFailed = this.adapter.status.every((s) => !s.available);\n    let lastRequestId: string = '';\n    let lastSegmentId: string = '';\n    if (allTTSFailed) {\n      this._logger.warn('All fallback TTS instances failed, retrying from first...');\n    }\n    for (let i = 0; i < this.adapter.ttsInstances.length; i++) {\n      const tts = this.adapter.ttsInstances[i]!;\n      const status = this.adapter.status[i]!;\n      if (!status.available && !allTTSFailed) {\n        this.adapter.markUnAvailable(i);\n        continue;\n      }\n      try {\n        this._logger.debug({ tts: tts.label }, 'attempting TTS synthesis');\n        const connOptions: APIConnectOptions = {\n          ...this.connOptions,\n          maxRetry: this.adapter.maxRetryPerTTS,\n        };\n        const stream = tts.synthesize(this.inputText, connOptions, this.abortSignal);\n        let audioReceived = false;\n        const resampler = this.adapter.createResamplerForTTS(i);\n        for await (const audio of stream) {\n          if (this.abortController.signal.aborted) {\n            stream.close();\n            return;\n          }\n\n          if (resampler) {\n            for (const frame of resampler.push(audio.frame)) {\n              this.queue.put({\n                ...audio,\n                frame,\n              });\n              audioReceived = true;\n            }\n          } else {\n            this.queue.put(audio);\n            audioReceived = true;\n          }\n          lastRequestId = audio.requestId;\n          lastSegmentId = audio.segmentId;\n        }\n\n        // Flush any remaining resampled frames\n        if (resampler) {\n          for (const frame of resampler.flush()) {\n            this.queue.put({\n              requestId: lastRequestId || '',\n              segmentId: lastSegmentId || '',\n              frame,\n              final: true,\n            });\n            audioReceived = true;\n          }\n        }\n\n        // Verify audio was actually received - silent failures should trigger fallback\n        if (!audioReceived) {\n          throw new APIConnectionError({\n            message: 'TTS synthesis completed but no audio was received',\n          });\n        }\n\n        this._logger.debug({ tts: tts.label }, 'TTS synthesis succeeded');\n        return;\n      } catch (error) {\n        if (error instanceof APIError || error instanceof APIConnectionError) {\n          this._logger.warn({ tts: tts.label, error }, 'TTS failed, switching to next instance');\n          this.adapter.markUnAvailable(i);\n        } else {\n          throw error;\n        }\n      }\n    }\n    const labels = this.adapter.ttsInstances.map((t) => t.label).join(', ');\n    throw new APIConnectionError({\n      message: `all TTS instances failed (${labels})`,\n    });\n  }\n}\n\nclass FallbackSynthesizeStream extends SynthesizeStream {\n  private adapter: FallbackAdapter;\n  private tokenBuffer: (\n    | string\n    | typeof SynthesizeStream.FLUSH_SENTINEL\n    | typeof SynthesizeStream.END_OF_STREAM\n  )[] = [];\n  private audioPushed = false;\n  private _logger = log();\n\n  label: string = 'tts.FallbackSynthesizeStream';\n\n  constructor(adapter: FallbackAdapter, connOptions: APIConnectOptions) {\n    super(adapter, connOptions);\n    this.adapter = adapter;\n  }\n\n  protected async run(): Promise<void> {\n    const allTTSFailed = this.adapter.status.every((s) => !s.available);\n    if (allTTSFailed) {\n      this._logger.warn('All fallback TTS instances failed, retrying from first...');\n    }\n    const readInputLLMStream = (async () => {\n      try {\n        for await (const input of this.input) {\n          if (this.abortController.signal.aborted) break;\n          this.tokenBuffer.push(input);\n        }\n      } catch (error) {\n        this._logger.debug({ error }, 'Error reading input LLM stream');\n        throw error;\n      } finally {\n        this.tokenBuffer.push(SynthesizeStream.END_OF_STREAM);\n      }\n    })();\n\n    for (let i = 0; i < this.adapter.ttsInstances.length; i++) {\n      const tts = this.adapter.getStreamingInstance(i);\n      const originalTts = this.adapter.ttsInstances[i]!;\n      const status = this.adapter.status[i]!;\n      let lastRequestId: string = '';\n      let lastSegmentId: string = '';\n\n      if (!status.available && !allTTSFailed) {\n        this.adapter.markUnAvailable(i);\n        continue;\n      }\n\n      try {\n        this._logger.debug({ tts: originalTts.label }, 'attempting TTS stream');\n\n        const connOptions: APIConnectOptions = {\n          ...this.connOptions,\n          maxRetry: this.adapter.maxRetryPerTTS,\n        };\n\n        const stream = tts.stream({ connOptions });\n        const resampler = this.adapter.createResamplerForTTS(i);\n        let bufferIndex = 0;\n        let streamOutputCompleted = false;\n        const forwardBufferToTTS = async () => {\n          while (true) {\n            while (bufferIndex < this.tokenBuffer.length) {\n              const token = this.tokenBuffer[bufferIndex++]!;\n              if (token === SynthesizeStream.FLUSH_SENTINEL) {\n                stream.flush();\n              } else if (token === SynthesizeStream.END_OF_STREAM) {\n                stream.endInput();\n                return;\n              } else {\n                stream.pushText(token);\n              }\n            }\n            await new Promise((resolve) => setTimeout(resolve, FORWARD_POLL_MS));\n            if (this.abortController.signal.aborted || streamOutputCompleted) {\n              stream.endInput();\n              return;\n            }\n          }\n        };\n\n        const processOutput = async () => {\n          try {\n            for await (const audio of stream) {\n              if (this.abortController.signal.aborted) {\n                stream.close();\n                return;\n              }\n\n              if (audio === SynthesizeStream.END_OF_STREAM) {\n                // Don't forward END_OF_STREAM yet — only emit after we verify audio\n                // was received. Otherwise a silent failure would signal completion\n                // to consumers before fallback can try the next TTS.\n                continue;\n              }\n\n              if (resampler) {\n                for (const frame of resampler.push(audio.frame)) {\n                  this.queue.put({\n                    ...audio,\n                    frame,\n                  });\n                  this.audioPushed = true;\n                }\n              } else {\n                this.queue.put(audio);\n                this.audioPushed = true;\n              }\n              lastRequestId = audio.requestId;\n              lastSegmentId = audio.segmentId;\n            }\n\n            // Flush resampler\n            if (resampler) {\n              for (const frame of resampler.flush()) {\n                this.queue.put({\n                  requestId: lastRequestId || '',\n                  segmentId: lastSegmentId || '',\n                  frame,\n                  final: true,\n                });\n                this.audioPushed = true;\n              }\n            }\n          } finally {\n            // processOutput and forwardBufferToTTS run in parallel.\n            // forwardBufferToTTS polls tokenBuffer and only exits when it sees END_OF_STREAM.\n            // But END_OF_STREAM is only added when the LLM finishes streaming (line 417).\n            // If the TTS fails while the LLM is still streaming, forwardBufferToTTS would\n            // keep polling indefinitely, blocking fallback to the next TTS.\n            // This flag tells it to exit early.\n            streamOutputCompleted = true;\n          }\n        };\n        const [outputResult, forwardBufferResult] = await Promise.allSettled([\n          processOutput(),\n          forwardBufferToTTS().catch((err) => {\n            stream.close(); // Close stream so processOutput can exit\n            throw err;\n          }),\n        ]);\n        if (outputResult.status === 'rejected') {\n          stream.close();\n          throw outputResult.reason;\n        }\n        if (forwardBufferResult.status === 'rejected') {\n          stream.close();\n          throw forwardBufferResult.reason;\n        }\n\n        // Verify audio was actually received - if not, the TTS failed silently\n        if (!this.audioPushed) {\n          throw new APIConnectionError({\n            message: 'TTS stream completed but no audio was received',\n          });\n        }\n\n        this.queue.put(SynthesizeStream.END_OF_STREAM);\n        this._logger.debug({ tts: originalTts.label }, 'TTS stream succeeded');\n        await readInputLLMStream.catch(() => {});\n        return;\n      } catch (error) {\n        if (this.audioPushed) {\n          this._logger.error(\n            { tts: originalTts.label },\n            'TTS failed after audio pushed, cannot fallback mid-utterance',\n          );\n          throw error;\n        }\n\n        if (error instanceof APIError || error instanceof APIConnectionError) {\n          this._logger.warn(\n            { tts: originalTts.label, error },\n            'TTS failed, switching to next instance',\n          );\n          this.adapter.markUnAvailable(i);\n        } else {\n          throw error;\n        }\n      }\n    }\n    await readInputLLMStream.catch(() => {});\n    const labels = this.adapter.ttsInstances.map((t) => t.label).join(', ');\n    throw new APIConnectionError({\n      message: `all TTS instances failed (${labels})`,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA+B;AAC/B,wBAA6C;AAC7C,iBAAoB;AACpB,sBAAsB;AACtB,mBAAoE;AACpE,mBAAoC;AACpC,4BAA8B;AAC9B,iBAA2E;AAiC3E,MAAM,uCAA0D;AAAA,EAC9D,UAAU;AAAA,EACV,WAAW,yCAA4B;AAAA,EACvC,iBAAiB,yCAA4B;AAC/C;AAEA,MAAM,kBAAkB;AA+BjB,MAAM,wBAAwB,eAAI;AAAA;AAAA,EAE9B;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAED,UAAuB,CAAC;AAAA,EACxB,cAAU,gBAAI;AAAA,EACd,oBAAiD,oBAAI,IAAI;AAAA,EAEjE,QAAgB;AAAA,EAEhB,YAAY,MAA8B;AACxC,QAAI,CAAC,KAAK,gBAAgB,KAAK,aAAa,SAAS,GAAG;AACtD,YAAM,IAAI,MAAM,6CAA6C;AAAA,IAC/D;AACA,UAAM,cAAc,KAAK,aAAa,CAAC,EAAG;AAC1C,UAAM,sBAAsB,KAAK,aAAa,MAAM,CAAC,QAAQ,IAAI,gBAAgB,WAAW;AAC5F,QAAI,CAAC,qBAAqB;AACxB,YAAM,IAAI,MAAM,2DAA2D;AAAA,IAC7E;AACA,UAAM,aAAa,KAAK,IAAI,GAAG,KAAK,aAAa,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC;AACzE,UAAM,eAAe,gBAAgB,sBAAsB,KAAK,YAAY;AAC5E,UAAM,YAAY,aAAa,YAAY;AAC3C,SAAK,eAAe,KAAK;AACzB,SAAK,iBAAiB,KAAK,kBAAkB;AAC7C,SAAK,kBAAkB,KAAK,mBAAmB;AAC/C,SAAK,UAAU,KAAK,aAAa,IAAI,OAAO;AAAA,MAC1C,WAAW;AAAA,MACX,gBAAgB;AAAA,IAClB,EAAE;AACF,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EACA,OAAe,sBAAsB,WAAmC;AACtE,UAAM,YAAY,UAAU,KAAK,CAAC,QAAQ,IAAI,aAAa,SAAS;AACpE,UAAM,oBAAoB,UAAU,MAAM,CAAC,QAAQ,IAAI,aAAa,sBAAsB,IAAI;AAC9F,WAAO,EAAE,WAAW,kBAAkB;AAAA,EACxC;AAAA,EAEQ,uBAA6B;AACnC,SAAK,aAAa,QAAQ,CAAC,QAAQ;AACjC,UAAI,GAAG,qBAAqB,CAAC,YAAY;AACvC,aAAK,KAAK,qBAAqB,OAAO;AAAA,MACxC,CAAC;AACD,UAAI,GAAG,SAAS,CAAC,UAAU;AACzB,aAAK,KAAK,SAAS,KAAK;AAAA,MAC1B,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,qBAAqB,OAAoB;AACvC,UAAM,MAAM,KAAK,aAAa,KAAK;AACnC,QAAI,IAAI,aAAa,WAAW;AAC9B,aAAO;AAAA,IACT;AAEA,WAAO,IAAI,oCAAc,KAAK,IAAI,sBAAM,kBAAkB,CAAC;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,sBAAsB,OAAsC;AAC1D,UAAM,MAAM,KAAK,aAAa,KAAK;AACnC,QAAI,KAAK,eAAe,IAAI,YAAY;AACtC,WAAK,QAAQ;AAAA,QACX,cAAc,IAAI,KAAK,SAAS,IAAI,UAAU,SAAS,KAAK,UAAU;AAAA,MACxE;AACA,aAAO,IAAI,+BAAe,IAAI,YAAY,KAAK,YAAY,IAAI,WAAW;AAAA,IAC5E;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,wBAAwB,KAAU,WAA0B;AAClE,UAAM,QAAkC,EAAE,KAAK,UAAU;AACzD,IAAC,KAAsF;AAAA,MACrF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,YAAY,OAAqB;AACvC,UAAM,SAAS,KAAK,QAAQ,KAAK;AACjC,UAAM,MAAM,KAAK,aAAa,KAAK;AACnC,QAAI,OAAO,kBAAkB,CAAC,OAAO,eAAe,MAAM;AACxD;AAAA,IACF;AACA,WAAO,iBAAiB,kBAAK,KAAK,OAAO,eAAe;AACtD,UAAI;AACF,cAAM,aAAa,IAAI;AAAA,UACrB;AAAA,UACA;AAAA,YACE,UAAU;AAAA,YACV,WAAW;AAAA,YACX,iBAAiB;AAAA,UACnB;AAAA,UACA,WAAW;AAAA,QACb;AACA,YAAI,gBAAgB;AACpB,yBAAiB,KAAK,YAAY;AAChC,0BAAgB;AAAA,QAClB;AACA,YAAI,CAAC,eAAe;AAClB,gBAAM,IAAI,MAAM,mDAAmD;AAAA,QACrE;AAEA,eAAO,YAAY;AACnB,eAAO,iBAAiB;AACxB,aAAK,QAAQ,KAAK,EAAE,KAAK,IAAI,MAAM,GAAG,eAAe;AACrD,aAAK,wBAAwB,KAAK,IAAI;AAAA,MACxC,SAAS,OAAO;AACd,eAAO,iBAAiB;AAExB,YAAI,WAAW,OAAO,SAAS;AAC7B;AAAA,QACF;AACA,aAAK,QAAQ,MAAM,EAAE,KAAK,IAAI,OAAO,MAAM,GAAG,iCAAiC;AAE/E,cAAM,YAAY,WAAW,MAAM;AACjC,eAAK,kBAAkB,OAAO,KAAK;AACnC,eAAK,YAAY,KAAK;AAAA,QACxB,GAAG,KAAK,eAAe;AACvB,aAAK,kBAAkB,IAAI,OAAO,SAAS;AAAA,MAC7C;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,gBAAgB,OAAqB;AACnC,UAAM,SAAS,KAAK,QAAQ,KAAK;AACjC,QAAI,OAAO,kBAAkB,CAAC,OAAO,eAAe,MAAM;AACxD;AAAA,IACF;AACA,QAAI,OAAO,WAAW;AACpB,aAAO,YAAY;AACnB,WAAK,wBAAwB,KAAK,aAAa,KAAK,GAAI,KAAK;AAAA,IAC/D;AACA,SAAK,YAAY,KAAK;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA,eAAe;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAO,SAAiE;AACtE,WAAO,IAAI;AAAA,MACT;AAAA,OACA,mCAAS,gBAAe;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,QAAuB;AAE3B,SAAK,kBAAkB,QAAQ,CAAC,cAAc;AAC5C,mBAAa,SAAS;AAAA,IACxB,CAAC;AACD,SAAK,kBAAkB,MAAM;AAG7B,UAAM,gBAAgB,KAAK,QACxB,IAAI,CAAC,MAAM,EAAE,cAAc,EAC3B,OAAO,CAAC,MAAuB,MAAM,IAAI;AAE5C,QAAI,cAAc,SAAS,GAAG;AAC5B,gBAAM,4BAAc,eAAe,GAAI;AAAA,IACzC;AAGA,eAAW,OAAO,KAAK,cAAc;AACnC,UAAI,mBAAmB,mBAAmB;AAC1C,UAAI,mBAAmB,OAAO;AAAA,IAChC;AAGA,UAAM,QAAQ,IAAI,KAAK,aAAa,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;AAAA,EAC/D;AACF;AAEA,MAAM,8BAA8B,yBAAc;AAAA,EACxC;AAAA,EACA;AAAA,EACA,cAAU,gBAAI;AAAA,EAEtB,QAAgB;AAAA,EAEhB,YACE,SACA,MACA,aACA,aACA;AACA,UAAM,MAAM,SAAS,aAAa,WAAW;AAC7C,SAAK,UAAU;AACf,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAgB,MAAqB;AACnC,UAAM,eAAe,KAAK,QAAQ,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS;AAClE,QAAI,gBAAwB;AAC5B,QAAI,gBAAwB;AAC5B,QAAI,cAAc;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAAA,IAC/E;AACA,aAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,aAAa,QAAQ,KAAK;AACzD,YAAM,MAAM,KAAK,QAAQ,aAAa,CAAC;AACvC,YAAM,SAAS,KAAK,QAAQ,OAAO,CAAC;AACpC,UAAI,CAAC,OAAO,aAAa,CAAC,cAAc;AACtC,aAAK,QAAQ,gBAAgB,CAAC;AAC9B;AAAA,MACF;AACA,UAAI;AACF,aAAK,QAAQ,MAAM,EAAE,KAAK,IAAI,MAAM,GAAG,0BAA0B;AACjE,cAAM,cAAiC;AAAA,UACrC,GAAG,KAAK;AAAA,UACR,UAAU,KAAK,QAAQ;AAAA,QACzB;AACA,cAAM,SAAS,IAAI,WAAW,KAAK,WAAW,aAAa,KAAK,WAAW;AAC3E,YAAI,gBAAgB;AACpB,cAAM,YAAY,KAAK,QAAQ,sBAAsB,CAAC;AACtD,yBAAiB,SAAS,QAAQ;AAChC,cAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC,mBAAO,MAAM;AACb;AAAA,UACF;AAEA,cAAI,WAAW;AACb,uBAAW,SAAS,UAAU,KAAK,MAAM,KAAK,GAAG;AAC/C,mBAAK,MAAM,IAAI;AAAA,gBACb,GAAG;AAAA,gBACH;AAAA,cACF,CAAC;AACD,8BAAgB;AAAA,YAClB;AAAA,UACF,OAAO;AACL,iBAAK,MAAM,IAAI,KAAK;AACpB,4BAAgB;AAAA,UAClB;AACA,0BAAgB,MAAM;AACtB,0BAAgB,MAAM;AAAA,QACxB;AAGA,YAAI,WAAW;AACb,qBAAW,SAAS,UAAU,MAAM,GAAG;AACrC,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,iBAAiB;AAAA,cAC5B,WAAW,iBAAiB;AAAA,cAC5B;AAAA,cACA,OAAO;AAAA,YACT,CAAC;AACD,4BAAgB;AAAA,UAClB;AAAA,QACF;AAGA,YAAI,CAAC,eAAe;AAClB,gBAAM,IAAI,qCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,aAAK,QAAQ,MAAM,EAAE,KAAK,IAAI,MAAM,GAAG,yBAAyB;AAChE;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,8BAAY,iBAAiB,sCAAoB;AACpE,eAAK,QAAQ,KAAK,EAAE,KAAK,IAAI,OAAO,MAAM,GAAG,wCAAwC;AACrF,eAAK,QAAQ,gBAAgB,CAAC;AAAA,QAChC,OAAO;AACL,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,UAAM,SAAS,KAAK,QAAQ,aAAa,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI;AACtE,UAAM,IAAI,qCAAmB;AAAA,MAC3B,SAAS,6BAA6B,MAAM;AAAA,IAC9C,CAAC;AAAA,EACH;AACF;AAEA,MAAM,iCAAiC,4BAAiB;AAAA,EAC9C;AAAA,EACA,cAIF,CAAC;AAAA,EACC,cAAc;AAAA,EACd,cAAU,gBAAI;AAAA,EAEtB,QAAgB;AAAA,EAEhB,YAAY,SAA0B,aAAgC;AACpE,UAAM,SAAS,WAAW;AAC1B,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,MAAgB,MAAqB;AACnC,UAAM,eAAe,KAAK,QAAQ,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS;AAClE,QAAI,cAAc;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAAA,IAC/E;AACA,UAAM,sBAAsB,YAAY;AACtC,UAAI;AACF,yBAAiB,SAAS,KAAK,OAAO;AACpC,cAAI,KAAK,gBAAgB,OAAO,QAAS;AACzC,eAAK,YAAY,KAAK,KAAK;AAAA,QAC7B;AAAA,MACF,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,EAAE,MAAM,GAAG,gCAAgC;AAC9D,cAAM;AAAA,MACR,UAAE;AACA,aAAK,YAAY,KAAK,4BAAiB,aAAa;AAAA,MACtD;AAAA,IACF,GAAG;AAEH,aAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,aAAa,QAAQ,KAAK;AACzD,YAAM,MAAM,KAAK,QAAQ,qBAAqB,CAAC;AAC/C,YAAM,cAAc,KAAK,QAAQ,aAAa,CAAC;AAC/C,YAAM,SAAS,KAAK,QAAQ,OAAO,CAAC;AACpC,UAAI,gBAAwB;AAC5B,UAAI,gBAAwB;AAE5B,UAAI,CAAC,OAAO,aAAa,CAAC,cAAc;AACtC,aAAK,QAAQ,gBAAgB,CAAC;AAC9B;AAAA,MACF;AAEA,UAAI;AACF,aAAK,QAAQ,MAAM,EAAE,KAAK,YAAY,MAAM,GAAG,uBAAuB;AAEtE,cAAM,cAAiC;AAAA,UACrC,GAAG,KAAK;AAAA,UACR,UAAU,KAAK,QAAQ;AAAA,QACzB;AAEA,cAAM,SAAS,IAAI,OAAO,EAAE,YAAY,CAAC;AACzC,cAAM,YAAY,KAAK,QAAQ,sBAAsB,CAAC;AACtD,YAAI,cAAc;AAClB,YAAI,wBAAwB;AAC5B,cAAM,qBAAqB,YAAY;AACrC,iBAAO,MAAM;AACX,mBAAO,cAAc,KAAK,YAAY,QAAQ;AAC5C,oBAAM,QAAQ,KAAK,YAAY,aAAa;AAC5C,kBAAI,UAAU,4BAAiB,gBAAgB;AAC7C,uBAAO,MAAM;AAAA,cACf,WAAW,UAAU,4BAAiB,eAAe;AACnD,uBAAO,SAAS;AAChB;AAAA,cACF,OAAO;AACL,uBAAO,SAAS,KAAK;AAAA,cACvB;AAAA,YACF;AACA,kBAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,eAAe,CAAC;AACnE,gBAAI,KAAK,gBAAgB,OAAO,WAAW,uBAAuB;AAChE,qBAAO,SAAS;AAChB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAEA,cAAM,gBAAgB,YAAY;AAChC,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,kBAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC,uBAAO,MAAM;AACb;AAAA,cACF;AAEA,kBAAI,UAAU,4BAAiB,eAAe;AAI5C;AAAA,cACF;AAEA,kBAAI,WAAW;AACb,2BAAW,SAAS,UAAU,KAAK,MAAM,KAAK,GAAG;AAC/C,uBAAK,MAAM,IAAI;AAAA,oBACb,GAAG;AAAA,oBACH;AAAA,kBACF,CAAC;AACD,uBAAK,cAAc;AAAA,gBACrB;AAAA,cACF,OAAO;AACL,qBAAK,MAAM,IAAI,KAAK;AACpB,qBAAK,cAAc;AAAA,cACrB;AACA,8BAAgB,MAAM;AACtB,8BAAgB,MAAM;AAAA,YACxB;AAGA,gBAAI,WAAW;AACb,yBAAW,SAAS,UAAU,MAAM,GAAG;AACrC,qBAAK,MAAM,IAAI;AAAA,kBACb,WAAW,iBAAiB;AAAA,kBAC5B,WAAW,iBAAiB;AAAA,kBAC5B;AAAA,kBACA,OAAO;AAAA,gBACT,CAAC;AACD,qBAAK,cAAc;AAAA,cACrB;AAAA,YACF;AAAA,UACF,UAAE;AAOA,oCAAwB;AAAA,UAC1B;AAAA,QACF;AACA,cAAM,CAAC,cAAc,mBAAmB,IAAI,MAAM,QAAQ,WAAW;AAAA,UACnE,cAAc;AAAA,UACd,mBAAmB,EAAE,MAAM,CAAC,QAAQ;AAClC,mBAAO,MAAM;AACb,kBAAM;AAAA,UACR,CAAC;AAAA,QACH,CAAC;AACD,YAAI,aAAa,WAAW,YAAY;AACtC,iBAAO,MAAM;AACb,gBAAM,aAAa;AAAA,QACrB;AACA,YAAI,oBAAoB,WAAW,YAAY;AAC7C,iBAAO,MAAM;AACb,gBAAM,oBAAoB;AAAA,QAC5B;AAGA,YAAI,CAAC,KAAK,aAAa;AACrB,gBAAM,IAAI,qCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,aAAK,MAAM,IAAI,4BAAiB,aAAa;AAC7C,aAAK,QAAQ,MAAM,EAAE,KAAK,YAAY,MAAM,GAAG,sBAAsB;AACrE,cAAM,mBAAmB,MAAM,MAAM;AAAA,QAAC,CAAC;AACvC;AAAA,MACF,SAAS,OAAO;AACd,YAAI,KAAK,aAAa;AACpB,eAAK,QAAQ;AAAA,YACX,EAAE,KAAK,YAAY,MAAM;AAAA,YACzB;AAAA,UACF;AACA,gBAAM;AAAA,QACR;AAEA,YAAI,iBAAiB,8BAAY,iBAAiB,sCAAoB;AACpE,eAAK,QAAQ;AAAA,YACX,EAAE,KAAK,YAAY,OAAO,MAAM;AAAA,YAChC;AAAA,UACF;AACA,eAAK,QAAQ,gBAAgB,CAAC;AAAA,QAChC,OAAO;AACL,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,UAAM,mBAAmB,MAAM,MAAM;AAAA,IAAC,CAAC;AACvC,UAAM,SAAS,KAAK,QAAQ,aAAa,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI;AACtE,UAAM,IAAI,qCAAmB;AAAA,MAC3B,SAAS,6BAA6B,MAAM;AAAA,IAC9C,CAAC;AAAA,EACH;AACF;","names":[]}